geo-rep: Fix ignoring geo-rep safe errors

Fix ignoring geo-rep safe errors in fuse layer and also ignore logging in client translator for mknod. Though it is rare, to happen with mknod, it might happen with history crawl on overlapping changelogs replay. Change-Id: I7e145cd1dc53f04d444ad2e68e66e648be448e61 BUG: 1210562 Signed-off-by: Kotresh HR <khiremat@redhat.com> Reviewed-on: http://review.gluster.org/10422 Tested-by: NetBSD Build System Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Aravinda VK <avishwan@redhat.com>
author: Kotresh HR <khiremat@redhat.com> 2015-04-28 18:09:29 +0530
committer: Vijay Bellur <vbellur@redhat.com> 2015-05-04 02:29:19 -0700
commit: 02223349a0a141d5bd09edcd571a703bb2e5a94d (patch)
tree: 3dc5060f5ff8ecf1d6d668d29d2305e7bc2deeb1 /libglusterfs/src/timer.h
parent: 20353cc323704292753ea6b7b0034362109fef76 (diff)
0 files changed, 0 insertions, 0 deletions
diff --git a/xlators/Makefile.am b/xlators/Makefile.am
index b1643d26c..f60fa85ce 100644
--- a/xlators/Makefile.am
+++ b/xlators/Makefile.am
@@ -1,3 +1,4 @@
-SUBDIRS = cluster storage protocol performance debug features encryption mount nfs mgmt system
+SUBDIRS = cluster storage protocol performance debug features encryption mount nfs mgmt system \
+          playground
 
 CLEANFILES =
diff --git a/xlators/bindings/python/src/Makefile.am b/xlators/bindings/python/src/Makefile.am
index c0b9141c6..90370d861 100644
--- a/xlators/bindings/python/src/Makefile.am
+++ b/xlators/bindings/python/src/Makefile.am
@@ -9,7 +9,7 @@ pythondir = $(xlatordir)/python
 
 python_so_SOURCES = python.c
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall \
+AM_CFLAGS = -fPIC $(GF_CPPFLAGS) -Wall \
 	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles \
 	$(PYTHON_CPPLAGS) -DGLUSTER_PYTHON_PATH=\"$(pythondir)\"
 
diff --git a/xlators/bindings/python/src/gluster.py b/xlators/bindings/python/src/gluster.py
index ee0eb1310..337c983ec 100644
--- a/xlators/bindings/python/src/gluster.py
+++ b/xlators/bindings/python/src/gluster.py
@@ -1,19 +1,12 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
+
+#  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
 #
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+
 from ctypes import *
 from glustertypes import *
 from glusterstack import *
diff --git a/xlators/bindings/python/src/glusterstack.py b/xlators/bindings/python/src/glusterstack.py
index ba24c8165..0c071ae98 100644
--- a/xlators/bindings/python/src/glusterstack.py
+++ b/xlators/bindings/python/src/glusterstack.py
@@ -1,19 +1,12 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
+
+#  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
 #
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+
 from ctypes import *
 from glustertypes import *
 
diff --git a/xlators/bindings/python/src/glustertypes.py b/xlators/bindings/python/src/glustertypes.py
index e9069d07c..98437d22e 100644
--- a/xlators/bindings/python/src/glustertypes.py
+++ b/xlators/bindings/python/src/glustertypes.py
@@ -1,19 +1,12 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
+
+#  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
 #
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+
 from ctypes import *
 import collections
 
diff --git a/xlators/bindings/python/src/python.c b/xlators/bindings/python/src/python.c
index 3310a2115..9b96790de 100644
--- a/xlators/bindings/python/src/python.c
+++ b/xlators/bindings/python/src/python.c
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2007-2010 Chris AtLee <chris@atlee.ca>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #include <Python.h>
 
 #ifndef _CONFIG_H
@@ -45,7 +35,7 @@ python_writev (call_frame_t *frame,
               xlator_t *this,
               fd_t *fd,
               struct iovec *vector,
-              int32_t count, 
+              int32_t count,
               off_t offset)
 {
   python_private_t *priv = (python_private_t *)this->private;
@@ -148,7 +138,7 @@ init (xlator_t *this)
   Py_InitializeEx(0);
 
   if (!this->children) {
-    gf_log ("python", GF_LOG_ERROR, 
+    gf_log ("python", GF_LOG_ERROR,
             "FATAL: python should have exactly one child");
     return -1;
   }
@@ -166,7 +156,7 @@ init (xlator_t *this)
   }
 
   priv->pInterp = Py_NewInterpreter();
-    
+
   // Adjust python's path
   PyObject *syspath = PySys_GetObject("path");
   PyObject *path = PyString_FromString(GLUSTER_PYTHON_PATH);
@@ -188,7 +178,7 @@ init (xlator_t *this)
   priv->pVectorType = PyObject_GetAttrString(priv->pGlusterModule, "iovec");
 
   gf_log("python", GF_LOG_DEBUG, "Loading script...%s", priv->scriptname);
-  
+
   priv->pScriptModule = AnonModule_FromFile(priv->scriptname);
   if (!priv->pScriptModule || PyErr_Occurred())
   {
@@ -217,7 +207,7 @@ init (xlator_t *this)
   return 0;
 }
 
-void 
+void
 fini (xlator_t *this)
 {
   python_private_t *priv = (python_private_t*)(this->private);
diff --git a/xlators/bindings/python/src/testxlator.py b/xlators/bindings/python/src/testxlator.py
index 507455c85..59a991dca 100644
--- a/xlators/bindings/python/src/testxlator.py
+++ b/xlators/bindings/python/src/testxlator.py
@@ -1,19 +1,12 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
+"""
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+"""
 
 """
 This is a test translator written in python.
diff --git a/xlators/cluster/afr/src/Makefile.am b/xlators/cluster/afr/src/Makefile.am
index ed0901813..35d18a6c0 100644
--- a/xlators/cluster/afr/src/Makefile.am
+++ b/xlators/cluster/afr/src/Makefile.am
@@ -1,21 +1,31 @@
 xlator_LTLIBRARIES = afr.la pump.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
-afr_common_source = afr-dir-read.c afr-dir-write.c afr-inode-read.c afr-inode-write.c afr-open.c afr-transaction.c afr-self-heal-data.c afr-self-heal-common.c afr-self-heal-metadata.c afr-self-heal-entry.c afr-self-heal-algorithm.c afr-lk-common.c afr-self-heald.c $(top_builddir)/xlators/lib/src/libxlator.c
+afr_common_source = afr-dir-read.c afr-dir-write.c afr-inode-read.c \
+	afr-inode-write.c afr-open.c afr-transaction.c afr-self-heal-data.c \
+	afr-self-heal-common.c afr-self-heal-metadata.c afr-self-heal-entry.c \
+	afr-self-heal-algorithm.c afr-lk-common.c afr-self-heald.c \
+	$(top_builddir)/xlators/lib/src/libxlator.c
 
-afr_la_LDFLAGS = -module -avoidversion
+afr_la_LDFLAGS = -module -avoid-version
 afr_la_SOURCES = $(afr_common_source) afr.c
 afr_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-pump_la_LDFLAGS = -module -avoidversion
+pump_la_LDFLAGS = -module -avoid-version
 pump_la_SOURCES =  $(afr_common_source) pump.c
 pump_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = afr.h afr-transaction.h afr-inode-write.h afr-inode-read.h afr-dir-read.h afr-dir-write.h afr-self-heal.h afr-self-heal-common.h afr-self-heal-algorithm.h pump.h afr-mem-types.h afr-common.c afr-self-heald.h $(top_builddir)/xlators/lib/src/libxlator.h $(top_builddir)/glusterfsd/src/glusterfsd.h
+noinst_HEADERS = afr.h afr-transaction.h afr-inode-write.h afr-inode-read.h \
+	afr-dir-read.h afr-dir-write.h afr-self-heal.h afr-self-heal-common.h \
+	afr-self-heal-algorithm.h pump.h afr-mem-types.h afr-common.c \
+	afr-self-heald.h $(top_builddir)/xlators/lib/src/libxlator.h \
+	$(top_builddir)/glusterfsd/src/glusterfsd.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	    -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/contrib/md5 -shared -nostartfiles $(GF_CFLAGS) \
-	    -I$(top_srcdir)/xlators/lib/src -I$(top_srcdir)/rpc/rpc-lib/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES =
 
diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index 22c741151..af01f2ef2 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -58,10 +49,9 @@
 #include "afr-self-heald.h"
 #include "pump.h"
 
-#define AFR_ICTX_OPENDIR_DONE_MASK     0x0000000200000000ULL
-#define AFR_ICTX_SPLIT_BRAIN_MASK      0x0000000100000000ULL
+#define AFR_ICTX_OPENDIR_DONE_MASK     0x0000000100000000ULL
 #define AFR_ICTX_READ_CHILD_MASK       0x00000000FFFFFFFFULL
-
+#define AFR_STATISTICS_HISTORY_SIZE    50
 int
 afr_lookup_done_success_action (call_frame_t *frame, xlator_t *this,
                                 gf_boolean_t fail_conflict);
@@ -92,6 +82,11 @@ afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req, const char *path)
                                 path, priv->pending_key[i]);
                 /* 3 = data+metadata+entry */
         }
+        ret = dict_set_int32 (xattr_req, GF_GFIDLESS_LOOKUP, 1);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "%s: failed to set gfidless "
+                        "lookup", path);
+        }
 }
 
 int
@@ -123,6 +118,13 @@ afr_lookup_xattr_req_prepare (afr_local_t *local, xlator_t *this,
                         loc->path, GLUSTERFS_ENTRYLK_COUNT);
         }
 
+        ret = dict_set_uint32 (local->xattr_req, GLUSTERFS_PARENT_ENTRYLK, 0);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: Unable to set dict value for %s",
+                        loc->path, GLUSTERFS_PARENT_ENTRYLK);
+        }
+
         ret = dict_get_ptr (local->xattr_req, "gfid-req", gfid_req);
         if (ret) {
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -200,60 +202,86 @@ out:
         return ret;
 }
 
-afr_inode_ctx_t*
-afr_inode_ctx_get_from_addr (uint64_t addr, int32_t child_count)
+void
+afr_inode_ctx_destroy (afr_inode_ctx_t *ctx)
 {
-        int             ret  = -1;
-        afr_inode_ctx_t *ctx = NULL;
-        size_t          size = 0;
+        if (!ctx)
+                return;
+        GF_FREE (ctx->fresh_children);
+        GF_FREE (ctx);
+}
 
-        GF_ASSERT (child_count > 0);
+afr_inode_ctx_t*
+__afr_inode_ctx_get (inode_t *inode, xlator_t *this)
+{
+        int             ret      = 0;
+        uint64_t        ctx_addr = 0;
+        afr_inode_ctx_t *ctx     = NULL;
+        afr_private_t   *priv    = NULL;
 
-        if (!addr) {
-                ctx = GF_CALLOC (1, sizeof (*ctx),
-                                 gf_afr_mt_inode_ctx_t);
-                if (!ctx)
-                        goto out;
-                size = sizeof (*ctx->fresh_children);
-                ctx->fresh_children = GF_CALLOC (child_count, size,
-                                                 gf_afr_mt_int32_t);
-                if (!ctx->fresh_children)
-                        goto out;
-        } else {
-                ctx = (afr_inode_ctx_t*) (long) addr;
+        priv = this->private;
+        ret = __inode_ctx_get (inode, this, &ctx_addr);
+        if (ret < 0)
+                ctx_addr = 0;
+        if (ctx_addr != 0) {
+                ctx = (afr_inode_ctx_t*) (long) ctx_addr;
+                goto out;
         }
-        ret = 0;
+        ctx = GF_CALLOC (1, sizeof (*ctx),
+                         gf_afr_mt_inode_ctx_t);
+        if (!ctx)
+                goto fail;
+        ctx->fresh_children = GF_CALLOC (priv->child_count,
+                                         sizeof (*ctx->fresh_children),
+                                         gf_afr_mt_int32_t);
+        if (!ctx->fresh_children)
+                goto fail;
+        ret = __inode_ctx_put (inode, this, (uint64_t)ctx);
+        if (ret) {
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "failed to "
+                                  "set the inode ctx (%s)",
+                                  uuid_utoa (inode->gfid));
+                goto fail;
+        }
+
 out:
-        if (ret && ctx) {
-                if (ctx->fresh_children)
-                        GF_FREE (ctx->fresh_children);
-                GF_FREE (ctx);
-                ctx = NULL;
+        return ctx;
+
+fail:
+        afr_inode_ctx_destroy (ctx);
+        return NULL;
+}
+
+afr_inode_ctx_t*
+afr_inode_ctx_get (inode_t *inode, xlator_t *this)
+{
+        afr_inode_ctx_t *ctx = NULL;
+
+        LOCK (&inode->lock);
+        {
+                ctx = __afr_inode_ctx_get (inode, this);
         }
+        UNLOCK (&inode->lock);
         return ctx;
 }
 
 void
-afr_inode_get_ctx (xlator_t *this, inode_t *inode, afr_inode_params_t *params)
+afr_inode_get_ctx_params (xlator_t *this, inode_t *inode,
+                          afr_inode_params_t *params)
 {
         GF_ASSERT (inode);
         GF_ASSERT (params);
 
-        int             ret = 0;
         afr_inode_ctx_t *ctx = NULL;
         afr_private_t   *priv = NULL;
         int             i = 0;
-        uint64_t        ctx_addr   = 0;
         int32_t         read_child = -1;
         int32_t         *fresh_children = NULL;
 
         priv = this->private;
         LOCK (&inode->lock);
         {
-                ret = __inode_ctx_get (inode, this, &ctx_addr);
-                if (ret < 0)
-                        goto unlock;
-                ctx = afr_inode_ctx_get_from_addr (ctx_addr, priv->child_count);
+                ctx = __afr_inode_ctx_get (inode, this);
                 if (!ctx)
                         goto unlock;
                 switch (params->op) {
@@ -272,12 +300,6 @@ afr_inode_get_ctx (xlator_t *this, inode_t *inode, afr_inode_params_t *params)
                         if (ctx->masks & AFR_ICTX_OPENDIR_DONE_MASK)
                                 params->u.value = _gf_true;
                         break;
-                case AFR_INODE_GET_SPLIT_BRAIN:
-                        params->u.value = _gf_false;
-                        if (ctx->masks & AFR_ICTX_SPLIT_BRAIN_MASK)
-                                params->u.value = _gf_true;
-                        ;
-                        break;
                 default:
                         GF_ASSERT (0);
                         break;
@@ -290,11 +312,16 @@ unlock:
 gf_boolean_t
 afr_is_split_brain (xlator_t *this, inode_t *inode)
 {
-        afr_inode_params_t params = {0};
+        afr_inode_ctx_t *ctx = NULL;
+        gf_boolean_t    spb  = _gf_false;
 
-        params.op = AFR_INODE_GET_SPLIT_BRAIN;
-        afr_inode_get_ctx (this, inode, &params);
-        return params.u.value;
+        ctx = afr_inode_ctx_get (inode, this);
+        if (!ctx)
+                goto out;
+        if ((ctx->mdata_spb == SPB) || (ctx->data_spb == SPB))
+                spb = _gf_true;
+out:
+        return spb;
 }
 
 gf_boolean_t
@@ -303,11 +330,10 @@ afr_is_opendir_done (xlator_t *this, inode_t *inode)
         afr_inode_params_t params = {0};
 
         params.op = AFR_INODE_GET_OPENDIR_DONE;
-        afr_inode_get_ctx (this, inode, &params);
+        afr_inode_get_ctx_params (this, inode, &params);
         return params.u.value;
 }
 
-
 int32_t
 afr_inode_get_read_ctx (xlator_t *this, inode_t *inode, int32_t *fresh_children)
 {
@@ -315,7 +341,7 @@ afr_inode_get_read_ctx (xlator_t *this, inode_t *inode, int32_t *fresh_children)
 
         params.op = AFR_INODE_GET_READ_CTX;
         params.u.read_ctx.children = fresh_children;
-        afr_inode_get_ctx (this, inode, &params);
+        afr_inode_get_ctx_params (this, inode, &params);
         return params.u.read_ctx.read_child;
 }
 
@@ -377,31 +403,14 @@ afr_inode_ctx_set_opendir_done (afr_inode_ctx_t *ctx)
 }
 
 void
-afr_inode_ctx_set_splitbrain (afr_inode_ctx_t *ctx, gf_boolean_t set)
-{
-        uint64_t        remaining_mask = 0;
-        uint64_t        mask = 0;
-
-        if (set) {
-                remaining_mask = (~AFR_ICTX_SPLIT_BRAIN_MASK & ctx->masks);
-                mask = (0xFFFFFFFFFFFFFFFFULL & AFR_ICTX_SPLIT_BRAIN_MASK);
-                ctx->masks = remaining_mask | mask;
-        } else {
-                ctx->masks = (~AFR_ICTX_SPLIT_BRAIN_MASK & ctx->masks);
-        }
-}
-
-void
-afr_inode_set_ctx (xlator_t *this, inode_t *inode, afr_inode_params_t *params)
+afr_inode_set_ctx_params (xlator_t *this, inode_t *inode,
+                          afr_inode_params_t *params)
 {
         GF_ASSERT (inode);
         GF_ASSERT (params);
 
-        int             ret = 0;
         afr_inode_ctx_t *ctx            = NULL;
         afr_private_t   *priv           = NULL;
-        uint64_t        ctx_addr        = 0;
-        gf_boolean_t    set             = _gf_false;
         int32_t         read_child      = -1;
         int32_t         *fresh_children = NULL;
         int32_t         *stale_children = NULL;
@@ -409,10 +418,7 @@ afr_inode_set_ctx (xlator_t *this, inode_t *inode, afr_inode_params_t *params)
         priv = this->private;
         LOCK (&inode->lock);
         {
-                ret = __inode_ctx_get (inode, this, &ctx_addr);
-                if (ret < 0)
-                        ctx_addr = 0;
-                ctx = afr_inode_ctx_get_from_addr (ctx_addr, priv->child_count);
+                ctx = __afr_inode_ctx_get (inode, this);
                 if (!ctx)
                         goto unlock;
                 switch (params->op) {
@@ -432,33 +438,26 @@ afr_inode_set_ctx (xlator_t *this, inode_t *inode, afr_inode_params_t *params)
                 case AFR_INODE_SET_OPENDIR_DONE:
                         afr_inode_ctx_set_opendir_done (ctx);
                         break;
-                case AFR_INODE_SET_SPLIT_BRAIN:
-                        set = params->u.value;
-                        afr_inode_ctx_set_splitbrain (ctx, set);
-                        break;
                 default:
                         GF_ASSERT (0);
                         break;
                 }
-                ret = __inode_ctx_put (inode, this, (uint64_t)ctx);
-                if (ret) {
-                        gf_log_callingfn (this->name, GF_LOG_ERROR, "failed to "
-                                          "set the inode ctx (%s)",
-                                          uuid_utoa (inode->gfid));
-                }
         }
 unlock:
         UNLOCK (&inode->lock);
 }
 
 void
-afr_set_split_brain (xlator_t *this, inode_t *inode, gf_boolean_t set)
+afr_set_split_brain (xlator_t *this, inode_t *inode, afr_spb_state_t mdata_spb,
+                     afr_spb_state_t data_spb)
 {
-        afr_inode_params_t      params = {0};
+        afr_inode_ctx_t *ctx = NULL;
 
-        params.op = AFR_INODE_SET_SPLIT_BRAIN;
-        params.u.value          = set;
-        afr_inode_set_ctx (this, inode, &params);
+        ctx = afr_inode_ctx_get (inode, this);
+        if (mdata_spb != DONT_KNOW)
+                ctx->mdata_spb = mdata_spb;
+        if (data_spb != DONT_KNOW)
+                ctx->data_spb = data_spb;
 }
 
 void
@@ -467,7 +466,7 @@ afr_set_opendir_done (xlator_t *this, inode_t *inode)
         afr_inode_params_t params = {0};
 
         params.op = AFR_INODE_SET_OPENDIR_DONE;
-        afr_inode_set_ctx (this, inode, &params);
+        afr_inode_set_ctx_params (this, inode, &params);
 }
 
 void
@@ -486,7 +485,7 @@ afr_inode_set_read_ctx (xlator_t *this, inode_t *inode, int32_t read_child,
         params.op = AFR_INODE_SET_READ_CTX;
         params.u.read_ctx.read_child     = read_child;
         params.u.read_ctx.children = fresh_children;
-        afr_inode_set_ctx (this, inode, &params);
+        afr_inode_set_ctx_params (this, inode, &params);
 }
 
 void
@@ -499,7 +498,7 @@ afr_inode_rm_stale_children (xlator_t *this, inode_t *inode,
 
         params.op = AFR_INODE_RM_STALE_CHILDREN;
         params.u.read_ctx.children = stale_children;
-        afr_inode_set_ctx (this, inode, &params);
+        afr_inode_set_ctx_params (this, inode, &params);
 }
 
 gf_boolean_t
@@ -543,6 +542,10 @@ afr_is_read_child (int32_t *success_children, int32_t *sources,
         gf_boolean_t             success_child = _gf_false;
         gf_boolean_t             source        = _gf_false;
 
+        if (child < 0) {
+                return _gf_false;
+        }
+
         GF_ASSERT (success_children);
         GF_ASSERT (child_count > 0);
 
@@ -559,29 +562,69 @@ out:
         return (success_child && source);
 }
 
+int32_t
+afr_hash_child (int32_t *success_children, int32_t child_count,
+                unsigned int hmode, uuid_t gfid)
+{
+        uuid_t  gfid_copy = {0,};
+        pid_t pid;
+
+        if (!hmode) {
+                return -1;
+        }
+
+        if (gfid) {
+               uuid_copy(gfid_copy,gfid);
+        }
+        if (hmode > 1) {
+                /*
+                 * Why getpid?  Because it's one of the cheapest calls
+                 * available - faster than gethostname etc. - and returns a
+                 * constant-length value that's sure to be shorter than a UUID.
+                 * It's still very unlikely to be the same across clients, so
+                 * it still provides good mixing.  We're not trying for
+                 * perfection here.  All we need is a low probability that
+                 * multiple clients won't converge on the same subvolume.
+                 */
+                pid = getpid();
+                memcpy (gfid_copy, &pid, sizeof(pid));
+        }
+
+        return SuperFastHash((char *)gfid_copy,
+                             sizeof(gfid_copy)) % child_count;
+}
+
 /* If sources is NULL the xattrs are assumed to be of source for all
  * success_children.
  */
 int
-afr_select_read_child_from_policy (int32_t *success_children, int32_t child_count,
-                                   int32_t prev_read_child,
-                                   int32_t config_read_child, int32_t *sources)
+afr_select_read_child_from_policy (int32_t *success_children,
+                                   int32_t child_count, int32_t prev_read_child,
+                                   int32_t config_read_child, int32_t *sources,
+                                   unsigned int hmode, uuid_t gfid)
 {
         int32_t                  read_child   = -1;
         int                      i            = 0;
 
         GF_ASSERT (success_children);
 
-        read_child = prev_read_child;
+        read_child = config_read_child;
         if (afr_is_read_child (success_children, sources, child_count,
                                read_child))
                 goto out;
 
-        read_child = config_read_child;
+        read_child = prev_read_child;
         if (afr_is_read_child (success_children, sources, child_count,
                                read_child))
                 goto out;
 
+        read_child = afr_hash_child (success_children, child_count,
+                                     hmode, gfid);
+        if (afr_is_read_child (success_children, sources, child_count,
+                               read_child)) {
+                goto out;
+        }
+
         for (i = 0; i < child_count; i++) {
                 read_child = success_children[i];
                 if (read_child < 0)
@@ -601,7 +644,7 @@ out:
 void
 afr_set_read_ctx_from_policy (xlator_t *this, inode_t *inode,
                               int32_t *fresh_children, int32_t prev_read_child,
-                              int32_t config_read_child)
+                              int32_t config_read_child, uuid_t gfid)
 {
         int                      read_child = -1;
         afr_private_t            *priv = NULL;
@@ -611,7 +654,8 @@ afr_set_read_ctx_from_policy (xlator_t *this, inode_t *inode,
                                                         priv->child_count,
                                                         prev_read_child,
                                                         config_read_child,
-                                                        NULL);
+                                                        NULL,
+                                                        priv->hash_mode, gfid);
         if (read_child >= 0)
                 afr_inode_set_read_ctx (this, inode, read_child,
                                         fresh_children);
@@ -667,8 +711,11 @@ afr_get_call_child (xlator_t *this, unsigned char *child_up, int32_t read_child,
         GF_ASSERT (call_child);
         GF_ASSERT (last_index);
         GF_ASSERT (fresh_children);
-        GF_ASSERT (read_child >= 0);
 
+        if (read_child < 0) {
+                ret = -EIO;
+                goto out;
+        }
         priv = this->private;
         *call_child = -1;
         *last_index = -1;
@@ -717,6 +764,13 @@ out:
 }
 
 void
+afr_xattr_array_destroy (dict_t **xattr, unsigned int child_count)
+{
+        afr_reset_xattr (xattr, child_count);
+        GF_FREE (xattr);
+}
+
+void
 afr_local_sh_cleanup (afr_local_t *local, xlator_t *this)
 {
         afr_self_heal_t *sh = NULL;
@@ -725,60 +779,51 @@ afr_local_sh_cleanup (afr_local_t *local, xlator_t *this)
         sh = &local->self_heal;
         priv = this->private;
 
-        if (sh->buf)
-                GF_FREE (sh->buf);
+        if (sh->data_sh_info && strcmp (sh->data_sh_info, ""))
+                GF_FREE (sh->data_sh_info);
+
+        if (sh->metadata_sh_info && strcmp (sh->metadata_sh_info, ""))
+                GF_FREE (sh->metadata_sh_info);
 
-        if (sh->parentbufs)
-                GF_FREE (sh->parentbufs);
+        GF_FREE (sh->buf);
+
+        GF_FREE (sh->parentbufs);
 
         if (sh->inode)
                 inode_unref (sh->inode);
 
-        if (sh->xattr) {
-                afr_reset_xattr (sh->xattr, priv->child_count);
-                GF_FREE (sh->xattr);
-        }
+        afr_xattr_array_destroy (sh->xattr, priv->child_count);
 
-        if (sh->child_errno)
-                GF_FREE (sh->child_errno);
+        GF_FREE (sh->child_errno);
 
         afr_matrix_cleanup (sh->pending_matrix, priv->child_count);
         afr_matrix_cleanup (sh->delta_matrix, priv->child_count);
 
-        if (sh->sources)
-                GF_FREE (sh->sources);
+        GF_FREE (sh->sources);
 
-        if (sh->success)
-                GF_FREE (sh->success);
+        GF_FREE (sh->success);
 
-        if (sh->locked_nodes)
-                GF_FREE (sh->locked_nodes);
+        GF_FREE (sh->locked_nodes);
 
         if (sh->healing_fd) {
                 fd_unref (sh->healing_fd);
                 sh->healing_fd = NULL;
         }
 
-        if (sh->linkname)
-                GF_FREE ((char *)sh->linkname);
+        GF_FREE ((char *)sh->linkname);
 
-        if (sh->success_children)
-                GF_FREE (sh->success_children);
+        GF_FREE (sh->success_children);
 
-        if (sh->fresh_children)
-                GF_FREE (sh->fresh_children);
+        GF_FREE (sh->fresh_children);
 
-        if (sh->fresh_parent_dirs)
-                GF_FREE (sh->fresh_parent_dirs);
+        GF_FREE (sh->fresh_parent_dirs);
 
         loc_wipe (&sh->parent_loc);
         loc_wipe (&sh->lookup_loc);
 
-        if (sh->checksum)
-                GF_FREE (sh->checksum);
+        GF_FREE (sh->checksum);
 
-        if (sh->write_needed)
-                GF_FREE (sh->write_needed);
+        GF_FREE (sh->write_needed);
         if (sh->healing_fd)
                 fd_unref (sh->healing_fd);
 }
@@ -787,27 +832,26 @@ afr_local_sh_cleanup (afr_local_t *local, xlator_t *this)
 void
 afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this)
 {
-        afr_private_t * priv = NULL;
+        afr_private_t *priv    = NULL;
+        int           i        = 0;
 
         priv = this->private;
 
         afr_matrix_cleanup (local->pending, priv->child_count);
+        afr_matrix_cleanup (local->transaction.txn_changelog,
+                            priv->child_count);
 
-        if (local->internal_lock.locked_nodes)
-                GF_FREE (local->internal_lock.locked_nodes);
-
-        if (local->internal_lock.inode_locked_nodes)
-                GF_FREE (local->internal_lock.inode_locked_nodes);
+        GF_FREE (local->internal_lock.locked_nodes);
 
-        if (local->internal_lock.entry_locked_nodes)
-                GF_FREE (local->internal_lock.entry_locked_nodes);
+        for (i = 0; local->internal_lock.inodelk[i].domain; i++) {
+                GF_FREE (local->internal_lock.inodelk[i].locked_nodes);
+        }
 
-        if (local->internal_lock.lower_locked_nodes)
-                GF_FREE (local->internal_lock.lower_locked_nodes);
+        GF_FREE (local->internal_lock.lower_locked_nodes);
 
+        afr_entry_lockee_cleanup (&local->internal_lock);
 
         GF_FREE (local->transaction.pre_op);
-        GF_FREE (local->transaction.child_errno);
         GF_FREE (local->transaction.eager_lock);
 
         GF_FREE (local->transaction.basename);
@@ -815,6 +859,8 @@ afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this)
 
         loc_wipe (&local->transaction.parent_loc);
         loc_wipe (&local->transaction.new_parent_loc);
+
+        GF_FREE (local->transaction.postop_piggybacked);
 }
 
 
@@ -844,17 +890,13 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this)
         if (local->dict)
                 dict_unref (local->dict);
 
-        if (local->child_up)
-                GF_FREE (local->child_up);
+	GF_FREE(local->replies);
 
-        if (local->child_errno)
-                GF_FREE (local->child_errno);
+        GF_FREE (local->child_up);
 
-        if (local->fresh_children)
-                GF_FREE (local->fresh_children);
+        GF_FREE (local->child_errno);
 
-        if (local->fd_open_on)
-                GF_FREE (local->fd_open_on);
+        GF_FREE (local->fresh_children);
 
         { /* lookup */
                 if (local->cont.lookup.xattrs) {
@@ -872,27 +914,23 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this)
                         inode_unref (local->cont.lookup.inode);
                 }
 
-                if (local->cont.lookup.postparents)
-                        GF_FREE (local->cont.lookup.postparents);
+                GF_FREE (local->cont.lookup.postparents);
 
-                if (local->cont.lookup.bufs)
-                        GF_FREE (local->cont.lookup.bufs);
+                GF_FREE (local->cont.lookup.bufs);
 
-                if (local->cont.lookup.success_children)
-                        GF_FREE (local->cont.lookup.success_children);
+                GF_FREE (local->cont.lookup.success_children);
 
-                if (local->cont.lookup.sources)
-                        GF_FREE (local->cont.lookup.sources);
+                GF_FREE (local->cont.lookup.sources);
+                afr_matrix_cleanup (local->cont.lookup.pending_matrix,
+                                    priv->child_count);
         }
 
         { /* getxattr */
-                if (local->cont.getxattr.name)
-                        GF_FREE (local->cont.getxattr.name);
+                GF_FREE (local->cont.getxattr.name);
         }
 
         { /* lk */
-                if (local->cont.lk.locked_nodes)
-                        GF_FREE (local->cont.lk.locked_nodes);
+                GF_FREE (local->cont.lk.locked_nodes);
         }
 
         { /* create */
@@ -947,14 +985,19 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this)
         }
 
         { /* opendir */
-                if (local->cont.opendir.checksum)
-                        GF_FREE (local->cont.opendir.checksum);
+                GF_FREE (local->cont.opendir.checksum);
         }
 
         { /* readdirp */
                 if (local->cont.readdir.dict)
                         dict_unref (local->cont.readdir.dict);
         }
+
+        if (local->xdata_req)
+                dict_unref (local->xdata_req);
+
+        if (local->xdata_rsp)
+                dict_unref (local->xdata_rsp);
 }
 
 
@@ -1037,6 +1080,88 @@ afr_update_loc_gfids (loc_t *loc, struct iatt *buf, struct iatt *postparent)
                 uuid_copy (loc->pargfid, postparent->ia_gfid);
 }
 
+/*
+ * Quota size xattrs are not maintained by afr. There is a
+ * possibility that they differ even when both the directory changelog xattrs
+ * suggest everything is fine. So if there is at least one 'source' check among
+ * the sources which has the maximum quota size. Otherwise check among all the
+ * available ones for maximum quota size. This way if there is a source and
+ * stale copies it always votes for the 'source'.
+ * */
+
+static void
+afr_handle_quota_size (afr_local_t *local, xlator_t *this,
+                       dict_t *rsp_dict)
+{
+        int32_t       *sources       = NULL;
+        dict_t        *xattr         = NULL;
+        data_t        *max_data      = NULL;
+        int64_t       max_quota_size = -1;
+        data_t        *data          = NULL;
+        int64_t       *size          = NULL;
+        int64_t       quota_size     = -1;
+        afr_private_t *priv          = NULL;
+        int           i              = 0;
+        int           ret            = -1;
+        gf_boolean_t  source_present = _gf_false;
+
+        priv    = this->private;
+        sources = local->cont.lookup.sources;
+
+        if (rsp_dict == NULL) {
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "%s: Invalid "
+                                  "response dictionary", local->loc.path);
+                return;
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (sources[i]) {
+                        source_present = _gf_true;
+                        break;
+                }
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                /*
+                 * If there is at least one source lets check
+                 * for maximum quota sizes among sources, otherwise take the
+                 * maximum of the ones present to be on the safer side.
+                 */
+                if (source_present && !sources[i])
+                        continue;
+
+                xattr = local->cont.lookup.xattrs[i];
+                if (!xattr)
+                        continue;
+
+                data = dict_get (xattr, QUOTA_SIZE_KEY);
+                if (!data)
+                        continue;
+
+                size = (int64_t*)data->data;
+                quota_size = ntoh64(*size);
+                gf_log (this->name, GF_LOG_DEBUG, "%s: %d, size: %"PRId64,
+                        local->loc.path, i, quota_size);
+                if (quota_size > max_quota_size) {
+                        if (max_data)
+                                data_unref (max_data);
+
+                        max_quota_size = quota_size;
+                        max_data = data_ref (data);
+                }
+        }
+
+        if (max_data) {
+                ret = dict_set (rsp_dict, QUOTA_SIZE_KEY, max_data);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s: Failed to set "
+                                "quota size", local->loc.path);
+                }
+
+                data_unref (max_data);
+        }
+}
+
 int
 afr_lookup_build_response_params (afr_local_t *local, xlator_t *this)
 {
@@ -1081,13 +1206,18 @@ afr_lookup_build_response_params (afr_local_t *local, xlator_t *this)
                 ret = -1;
                 goto out;
         }
+
         gf_log (this->name, GF_LOG_DEBUG, "Building lookup response from %d",
                 read_child);
         if (!*xattr)
                 *xattr = dict_ref (local->cont.lookup.xattrs[read_child]);
+
         *buf = local->cont.lookup.bufs[read_child];
         *postparent = local->cont.lookup.postparents[read_child];
 
+        if (dict_get (local->xattr_req, QUOTA_SIZE_KEY))
+                afr_handle_quota_size (local, this, *xattr);
+
         if (IA_INVAL == local->cont.lookup.inode->ia_type) {
                 /* fix for RT #602 */
                 local->cont.lookup.inode->ia_type = buf->ia_type;
@@ -1103,6 +1233,7 @@ afr_lookup_update_lk_counts (afr_local_t *local, xlator_t *this,
         uint32_t inodelk_count = 0;
         uint32_t entrylk_count = 0;
         int      ret           = -1;
+        uint32_t parent_entrylk = 0;
 
         GF_ASSERT (local);
         GF_ASSERT (this);
@@ -1118,43 +1249,103 @@ afr_lookup_update_lk_counts (afr_local_t *local, xlator_t *this,
                                &entrylk_count);
         if (ret == 0)
                 local->entrylk_count += entrylk_count;
+        ret = dict_get_uint32 (xattr, GLUSTERFS_PARENT_ENTRYLK,
+                               &parent_entrylk);
+        if (!ret)
+                local->cont.lookup.parent_entrylk += parent_entrylk;
 }
 
+/*
+ * It's important to maintain a commutative property on do_*_self_heal and
+ * found*; once set, they must not be cleared by a subsequent iteration or
+ * call, so that they represent a logical OR of all iterations and calls
+ * regardless of child/key order.  That allows the caller to call us multiple
+ * times without having to use a separate variable as a "reduce" accumulator.
+ */
 static void
 afr_lookup_set_self_heal_params_by_xattr (afr_local_t *local, xlator_t *this,
                                           dict_t *xattr)
 {
+        afr_private_t *priv        = NULL;
+        int            i           = 0;
+        int            ret         = -1;
+        void          *pending_raw = NULL;
+        int32_t       *pending     = NULL;
+
         GF_ASSERT (local);
         GF_ASSERT (this);
         GF_ASSERT (xattr);
 
-        if (afr_sh_has_metadata_pending (xattr, this)) {
-                local->self_heal.do_metadata_self_heal = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "metadata self-heal is pending for %s.",
-                       local->loc.path);
-        }
+        priv = this->private;
 
-        if (afr_sh_has_entry_pending (xattr, this)) {
-                local->self_heal.do_entry_self_heal = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "entry self-heal is pending for %s.", local->loc.path);
-        }
+        for (i = 0; i < priv->child_count; i++) {
+                ret = dict_get_ptr (xattr, priv->pending_key[i],
+                                    &pending_raw);
+                if (ret != 0) {
+                        continue;
+                }
+                pending = pending_raw;
 
-        if (afr_sh_has_data_pending (xattr, this)) {
-                local->self_heal.do_data_self_heal = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "data self-heal is pending for %s.", local->loc.path);
+                if (pending[AFR_METADATA_TRANSACTION]) {
+                        gf_log(this->name, GF_LOG_DEBUG,
+                               "metadata self-heal is pending for %s.",
+                               local->loc.path);
+                        local->self_heal.do_metadata_self_heal = _gf_true;
+                }
+
+                if (pending[AFR_ENTRY_TRANSACTION]) {
+                        gf_log(this->name, GF_LOG_DEBUG,
+                               "entry self-heal is pending for %s.",
+                               local->loc.path);
+                        local->self_heal.do_entry_self_heal = _gf_true;
+                }
+
+                if (pending[AFR_DATA_TRANSACTION]) {
+                        gf_log(this->name, GF_LOG_DEBUG,
+                               "data self-heal is pending for %s.",
+                               local->loc.path);
+                        local->self_heal.do_data_self_heal = _gf_true;
+                }
         }
 }
 
+void
+afr_lookup_check_set_metadata_split_brain (afr_local_t *local, xlator_t *this)
+{
+        int32_t                  *sources = NULL;
+        afr_private_t            *priv = NULL;
+        int32_t                  subvol_status = 0;
+        int32_t                  *success_children   = NULL;
+        dict_t                   **xattrs = NULL;
+        struct iatt              *bufs = NULL;
+        int32_t                  **pending_matrix = NULL;
+
+        priv = this->private;
+
+        sources = GF_CALLOC (priv->child_count, sizeof (*sources),
+                             gf_afr_mt_int32_t);
+        if (NULL == sources)
+                goto out;
+        success_children = local->cont.lookup.success_children;
+        xattrs = local->cont.lookup.xattrs;
+        bufs = local->cont.lookup.bufs;
+        pending_matrix = local->cont.lookup.pending_matrix;
+        afr_build_sources (this, xattrs, bufs, pending_matrix,
+                           sources, success_children, AFR_METADATA_TRANSACTION,
+                           &subvol_status, _gf_false);
+        if (subvol_status & SPLIT_BRAIN)
+                local->cont.lookup.possible_spb = _gf_true;
+out:
+        GF_FREE (sources);
+}
+
 static void
 afr_detect_self_heal_by_iatt (afr_local_t *local, xlator_t *this,
                             struct iatt *buf, struct iatt *lookup_buf)
 {
         if (PERMISSION_DIFFERS (buf, lookup_buf)) {
                 /* mismatching permissions */
-                gf_log (this->name, GF_LOG_INFO,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "permissions differ for %s ", local->loc.path);
                 local->self_heal.do_metadata_self_heal = _gf_true;
         }
@@ -1162,27 +1353,45 @@ afr_detect_self_heal_by_iatt (afr_local_t *local, xlator_t *this,
         if (OWNERSHIP_DIFFERS (buf, lookup_buf)) {
                 /* mismatching permissions */
                 local->self_heal.do_metadata_self_heal = _gf_true;
-                gf_log (this->name, GF_LOG_INFO,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "ownership differs for %s ", local->loc.path);
         }
 
         if (SIZE_DIFFERS (buf, lookup_buf)
             && IA_ISREG (buf->ia_type)) {
-                gf_log (this->name, GF_LOG_INFO,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "size differs for %s ", local->loc.path);
                 local->self_heal.do_data_self_heal = _gf_true;
         }
 
         if (uuid_compare (buf->ia_gfid, lookup_buf->ia_gfid)) {
                 /* mismatching gfid */
-                gf_log (this->name, GF_LOG_WARNING,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "%s: gfid different on subvolume", local->loc.path);
         }
 }
 
 static void
-afr_detect_self_heal_by_lookup_status (afr_local_t *local, xlator_t *this,
-                                       gf_boolean_t split_brain)
+afr_detect_self_heal_by_split_brain_status (afr_local_t *local, xlator_t *this)
+{
+        gf_boolean_t split_brain = _gf_false;
+        afr_self_heal_t *sh = NULL;
+
+        sh = &local->self_heal;
+
+        split_brain = afr_is_split_brain (this, local->cont.lookup.inode);
+        split_brain = split_brain || local->cont.lookup.possible_spb;
+        if ((local->success_count > 0) && split_brain &&
+            IA_ISREG (local->cont.lookup.inode->ia_type)) {
+                sh->force_confirm_spb = _gf_true;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "split brain detected during lookup of %s.",
+                        local->loc.path);
+        }
+}
+
+static void
+afr_detect_self_heal_by_lookup_status (afr_local_t *local, xlator_t *this)
 {
         GF_ASSERT (local);
         GF_ASSERT (this);
@@ -1193,24 +1402,11 @@ afr_detect_self_heal_by_lookup_status (afr_local_t *local, xlator_t *this,
                 local->self_heal.do_entry_self_heal    = _gf_true;
                 local->self_heal.do_gfid_self_heal    = _gf_true;
                 local->self_heal.do_missing_entry_self_heal    = _gf_true;
-                gf_log(this->name, GF_LOG_INFO,
+                gf_log(this->name, GF_LOG_DEBUG,
                        "entries are missing in lookup of %s.",
                        local->loc.path);
-                //If all self-heals are needed no need to check for other rules
-                goto out;
         }
 
-        if ((local->success_count > 0) && split_brain &&
-            IA_ISREG (local->cont.lookup.inode->ia_type)) {
-                local->self_heal.do_data_self_heal = _gf_true;
-                local->self_heal.do_gfid_self_heal    = _gf_true;
-                local->self_heal.do_missing_entry_self_heal    = _gf_true;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "split brain detected during lookup of %s.",
-                        local->loc.path);
-        }
-
-out:
         return;
 }
 
@@ -1220,6 +1416,8 @@ afr_can_self_heal_proceed (afr_self_heal_t *sh, afr_private_t *priv)
         GF_ASSERT (sh);
         GF_ASSERT (priv);
 
+        if (sh->force_confirm_spb)
+                return _gf_true;
         return (sh->do_gfid_self_heal
                 || sh->do_missing_entry_self_heal
                 || (afr_data_self_heal_enabled (priv->data_self_heal) &&
@@ -1253,6 +1451,7 @@ afr_lookup_select_read_child (afr_local_t *local, xlator_t *this,
         dict_t                  **xattrs       = NULL;
         int32_t                 *success_children = NULL;
         afr_transaction_type    type           = AFR_METADATA_TRANSACTION;
+        uuid_t                  *gfid          = NULL;
 
         GF_ASSERT (local);
         GF_ASSERT (this);
@@ -1266,8 +1465,9 @@ afr_lookup_select_read_child (afr_local_t *local, xlator_t *this,
         ia_type = local->cont.lookup.bufs[success_children[0]].ia_type;
         type = afr_transaction_type_get (ia_type);
         xattrs = local->cont.lookup.xattrs;
+        gfid = &local->cont.lookup.buf.ia_gfid;
         source = afr_lookup_select_read_child_by_txn_type (this, local, xattrs,
-                                                           type);
+                                                           type, *gfid);
         if (source < 0) {
                 gf_log (this->name, GF_LOG_DEBUG, "failed to select source "
                         "for %s", local->loc.path);
@@ -1295,7 +1495,8 @@ afr_launch_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode,
                       void (*gfid_sh_success_cbk) (call_frame_t *sh_frame,
                                                    xlator_t *this),
                       int (*unwind) (call_frame_t *frame, xlator_t *this,
-                                     int32_t op_ret, int32_t op_errno))
+                                     int32_t op_ret, int32_t op_errno,
+                                     int32_t sh_failed))
 {
         afr_local_t             *local = NULL;
         char                    sh_type_str[256] = {0,};
@@ -1318,7 +1519,7 @@ afr_launch_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode,
 
         if (background)
                 bg = "background";
-        gf_log (this->name, GF_LOG_INFO,
+        gf_log (this->name, GF_LOG_DEBUG,
                 "%s %s self-heal triggered. path: %s, reason: %s", bg,
                 sh_type_str, local->loc.path, reason);
 
@@ -1389,7 +1590,7 @@ afr_conflicting_iattrs (struct iatt *bufs, int32_t *success_children,
 
                 child2 = &bufs[success_children[i-1]];
                 if (FILETYPE_DIFFERS (child1, child2)) {
-                        gf_log (xlator_name, GF_LOG_WARNING, "%s: filetype "
+                        gf_log (xlator_name, GF_LOG_DEBUG, "%s: filetype "
                                 "differs on subvolumes (%d, %d)", path,
                                 success_children[i-1], success_children[i]);
                         conflicting = _gf_true;
@@ -1398,7 +1599,7 @@ afr_conflicting_iattrs (struct iatt *bufs, int32_t *success_children,
                 if (!gfid || uuid_is_null (child1->ia_gfid))
                         continue;
                 if (uuid_compare (*gfid, child1->ia_gfid)) {
-                       gf_log (xlator_name, GF_LOG_WARNING, "%s: gfid differs"
+                       gf_log (xlator_name, GF_LOG_DEBUG, "%s: gfid differs"
                                " on subvolume %d", path, success_children[i]);
                        conflicting = _gf_true;
                        goto out;
@@ -1481,13 +1682,11 @@ afr_lookup_set_self_heal_params (afr_local_t *local, xlator_t *this)
         int32_t                 child1 = -1;
         int32_t                 child2 = -1;
         afr_self_heal_t         *sh = NULL;
-        gf_boolean_t            split_brain = _gf_false;
 
         priv  = this->private;
         sh = &local->self_heal;
 
-        split_brain = afr_is_split_brain (this, local->cont.lookup.inode);
-        afr_detect_self_heal_by_lookup_status (local, this, split_brain);
+        afr_detect_self_heal_by_lookup_status (local, this);
 
         if (afr_lookup_gfid_missing_count (local, this))
                 local->self_heal.do_gfid_self_heal    = _gf_true;
@@ -1514,23 +1713,28 @@ afr_lookup_set_self_heal_params (afr_local_t *local, xlator_t *this)
                 afr_lookup_set_self_heal_params_by_xattr (local, this,
                                                           xattr[child1]);
         }
-        if (afr_open_only_data_self_heal (priv->data_self_heal)
-            && !split_brain)
+        if (afr_open_only_data_self_heal (priv->data_self_heal))
                 sh->do_data_self_heal = _gf_false;
+        if (sh->do_metadata_self_heal)
+                afr_lookup_check_set_metadata_split_brain (local, this);
+        afr_detect_self_heal_by_split_brain_status (local, this);
 }
 
 int
 afr_self_heal_lookup_unwind (call_frame_t *frame, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno)
+                             int32_t op_ret, int32_t op_errno,
+                             int32_t sh_failed)
 {
         afr_local_t *local = NULL;
+        int         ret    = -1;
+        dict_t      *xattr = NULL;
 
         local = frame->local;
 
         if (op_ret == -1) {
                 local->op_ret = -1;
-                if (afr_error_more_important (local->op_errno, op_errno))
-                        local->op_errno = op_errno;
+		local->op_errno = afr_most_important_error(local->op_errno,
+							   op_errno, _gf_true);
 
                 goto out;
         } else {
@@ -1538,6 +1742,23 @@ afr_self_heal_lookup_unwind (call_frame_t *frame, xlator_t *this,
         }
 
         afr_lookup_done_success_action (frame, this, _gf_true);
+        xattr = local->cont.lookup.xattr;
+        if (xattr) {
+                ret = dict_set_int32 (xattr, "sh-failed", sh_failed);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR, "%s: Failed to set "
+                                "sh-failed to %d", local->loc.path, sh_failed);
+
+                if (local->self_heal.actual_sh_started == _gf_true &&
+                    sh_failed == 0) {
+                        ret = dict_set_int32 (xattr, "actual-sh-done", 1);
+                        if (ret)
+                                gf_log(this->name, GF_LOG_ERROR, "%s: Failed to"
+                                       " set actual-sh-done to %d",
+                                       local->loc.path,
+                                       local->self_heal.actual_sh_started);
+                }
+        }
 out:
         AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
                           local->cont.lookup.inode, &local->cont.lookup.buf,
@@ -1611,7 +1832,8 @@ afr_lookup_perform_self_heal (call_frame_t *frame, xlator_t *this,
 
         afr_lookup_set_self_heal_params (local, this);
         if (afr_can_self_heal_proceed (&local->self_heal, priv)) {
-                if  (afr_is_transaction_running (local))
+                if  (afr_is_transaction_running (local) &&
+                     (!local->allow_sh_for_running_transaction))
                         goto out;
 
                 reason = "lookup detected pending operations";
@@ -1672,20 +1894,15 @@ afr_lookup_done_success_action (call_frame_t *frame, xlator_t *this,
         int32_t             read_child = -1;
         int32_t             ret        = -1;
         afr_local_t         *local     = NULL;
-        afr_private_t       *priv      = NULL;
         gf_boolean_t        fresh_lookup = _gf_false;
 
         local   = frame->local;
-        priv    = this->private;
         fresh_lookup = local->cont.lookup.fresh_lookup;
 
         if (local->loc.parent == NULL)
                 fail_conflict = _gf_true;
 
-        if (afr_conflicting_iattrs (local->cont.lookup.bufs,
-                                     local->cont.lookup.success_children,
-                                     priv->child_count, local->loc.path,
-                                     this->name)) {
+        if (afr_lookup_conflicting_entries (local, this)) {
                 if (fail_conflict == _gf_false)
                         ret = 0;
                 goto out;
@@ -1717,6 +1934,135 @@ out:
         return ret;
 }
 
+int
+afr_lookup_get_latest_subvol (afr_local_t *local, xlator_t *this)
+{
+        afr_private_t *priv = NULL;
+        int32_t       *success_children = NULL;
+        struct iatt   *bufs = NULL;
+        int           i = 0;
+        int           child = 0;
+        int           lsubvol = -1;
+
+        priv = this->private;
+        success_children = local->cont.lookup.success_children;
+        bufs = local->cont.lookup.bufs;
+        for (i = 0; i < priv->child_count; i++) {
+                child = success_children[i];
+                if (child == -1)
+                        break;
+                if (uuid_is_null (bufs[child].ia_gfid))
+                        continue;
+                if (lsubvol < 0) {
+                        lsubvol = child;
+                } else if (bufs[lsubvol].ia_ctime < bufs[child].ia_ctime) {
+                        lsubvol = child;
+                } else if ((bufs[lsubvol].ia_ctime == bufs[child].ia_ctime) &&
+                  (bufs[lsubvol].ia_ctime_nsec < bufs[child].ia_ctime_nsec)) {
+                        lsubvol = child;
+                }
+        }
+        return lsubvol;
+}
+
+void
+afr_lookup_mark_other_entries_stale (afr_local_t *local, xlator_t *this,
+                                     int subvol)
+{
+        afr_private_t *priv = NULL;
+        int32_t       *success_children = NULL;
+        struct iatt   *bufs = NULL;
+        int           i = 0;
+        int           child = 0;
+
+        priv = this->private;
+        success_children = local->cont.lookup.success_children;
+        bufs = local->cont.lookup.bufs;
+        memcpy (local->fresh_children, success_children,
+                sizeof (*success_children) * priv->child_count);
+        for (i = 0; i < priv->child_count; i++) {
+                child = local->fresh_children[i];
+                if (child == -1)
+                        break;
+                if (child == subvol)
+                        continue;
+                if (uuid_is_null (bufs[child].ia_gfid) &&
+                    (bufs[child].ia_type == bufs[subvol].ia_type))
+                        continue;
+                afr_children_rm_child (success_children, child,
+                                       priv->child_count);
+                local->success_count--;
+        }
+        afr_reset_children (local->fresh_children, priv->child_count);
+}
+
+void
+afr_succeed_lookup_on_latest_iatt (afr_local_t *local, xlator_t *this)
+{
+        int    lsubvol = 0;
+
+        if (!afr_lookup_conflicting_entries (local, this))
+                goto out;
+
+        lsubvol = afr_lookup_get_latest_subvol (local, this);
+        if (lsubvol < 0)
+                goto out;
+        afr_lookup_mark_other_entries_stale (local, this, lsubvol);
+out:
+        return;
+}
+
+gf_boolean_t
+afr_is_entry_possibly_under_creation (afr_local_t *local, xlator_t *this)
+{
+        /*
+         * We need to perform this test in lookup done and treat on going
+         * create/DELETE as ENOENT.
+         * Reason:
+        Multiple clients A, B and C are attempting 'mkdir -p /mnt/a/b/c'
+
+        1 Client A is in the middle of mkdir(/a). It has acquired lock.
+          It has performed mkdir(/a) on one subvol, and second one is still
+          in progress
+        2 Client B performs a lookup, sees directory /a on one,
+          ENOENT on the other, succeeds lookup.
+        3 Client B performs lookup on /a/b on both subvols, both return ENOENT
+          (one subvol because /a/b does not exist, another because /a
+          itself does not exist)
+        4 Client B proceeds to mkdir /a/b. It obtains entrylk on inode=/a with
+          basename=b on one subvol, but fails on other subvol as /a is yet to
+          be created by Client A.
+        5 Client A finishes mkdir of /a on other subvol
+        6 Client C also attempts to create /a/b, lookup returns ENOENT on
+          both subvols.
+        7 Client C tries to obtain entrylk on on inode=/a with basename=b,
+          obtains on one subvol (where B had failed), and waits for B to unlock
+          on other subvol.
+        8 Client B finishes mkdir() on one subvol with GFID-1 and completes
+          transaction and unlocks
+        9 Client C gets the lock on the second subvol, At this stage second
+          subvol already has /a/b created from Client B, but Client C does not
+          check that in the middle of mkdir transaction
+        10 Client C attempts mkdir /a/b on both subvols. It succeeds on
+           ONLY ONE (where Client B could not get lock because of
+           missing parent /a dir) with GFID-2, and gets EEXIST from ONE subvol.
+        This way we have /a/b in GFID mismatch. One subvol got GFID-1 because
+        Client B performed transaction on only one subvol (because entrylk()
+        could not be obtained on second subvol because of missing parent dir --
+        caused by premature/speculative succeeding of lookup() on /a when locks
+        are detected). Other subvol gets GFID-2 from Client C because while
+        it was waiting for entrylk() on both subvols, Client B was in the
+        middle of creating mkdir() on only one subvol, and Client C does not
+        "expect" this when it is between lock() and pre-op()/op() phase of the
+        transaction.
+         */
+	if (local->cont.lookup.parent_entrylk && local->enoent_count)
+		return _gf_true;
+
+	return _gf_false;
+}
+
+
 static void
 afr_lookup_done (call_frame_t *frame, xlator_t *this)
 {
@@ -1733,8 +2079,18 @@ afr_lookup_done (call_frame_t *frame, xlator_t *this)
         priv  = this->private;
         local = frame->local;
 
+	if (afr_is_entry_possibly_under_creation (local, this)) {
+		local->op_ret = -1;
+		local->op_errno = ENOENT;
+		goto unwind;
+	}
+
         if (local->op_ret < 0)
                 goto unwind;
+
+        if (local->cont.lookup.parent_entrylk && local->success_count > 1)
+                afr_succeed_lookup_on_latest_iatt (local, this);
+
         gfid_miss_count = afr_lookup_gfid_missing_count (local, this);
         up_children_count = afr_up_children_count (local->child_up,
                                                    priv->child_count);
@@ -1786,25 +2142,20 @@ afr_lookup_done (call_frame_t *frame, xlator_t *this)
  * others in that they must be given higher priority while
  * returning to the user.
  *
- * The hierarchy is ESTALE > ENOENT > others
- *
+ * The hierarchy is ESTALE > EIO > ENOENT > others
  */
-
-gf_boolean_t
-afr_error_more_important (int32_t old_errno, int32_t new_errno)
+int32_t
+afr_most_important_error(int32_t old_errno, int32_t new_errno,
+			 gf_boolean_t eio)
 {
-        gf_boolean_t ret = _gf_true;
-
-        /* Nothing should ever overwrite ESTALE */
-        if (old_errno == ESTALE)
-                ret = _gf_false;
-
-        /* Nothing should overwrite ENOENT, except ESTALE/EIO*/
-        else if ((old_errno == ENOENT) && (new_errno != ESTALE)
-                 && (new_errno != EIO))
-                ret = _gf_false;
-
-        return ret;
+	if (old_errno == ESTALE || new_errno == ESTALE)
+		return ESTALE;
+	if (eio && (old_errno == EIO || new_errno == EIO))
+		return EIO;
+	if (old_errno == ENOENT || new_errno == ENOENT)
+		return ENOENT;
+
+	return new_errno;
 }
 
 int32_t
@@ -1823,8 +2174,9 @@ afr_resultant_errno_get (int32_t *children,
                 } else {
                         child = i;
                 }
-                if (afr_error_more_important (op_errno, child_errno[child]))
-                                op_errno = child_errno[child];
+		op_errno = afr_most_important_error(op_errno,
+						    child_errno[child],
+						    _gf_false);
         }
         return op_errno;
 }
@@ -1836,8 +2188,8 @@ afr_lookup_handle_error (afr_local_t *local, int32_t op_ret,  int32_t op_errno)
         if (op_errno == ENOENT)
                 local->enoent_count++;
 
-        if (afr_error_more_important (local->op_errno, op_errno))
-                local->op_errno = op_errno;
+	local->op_errno = afr_most_important_error(local->op_errno, op_errno,
+						   _gf_false);
 
         if (local->op_errno == ESTALE) {
                 local->op_ret = -1;
@@ -1884,12 +2236,79 @@ afr_lookup_handle_first_success (afr_local_t *local, xlator_t *this,
         afr_set_root_inode_on_first_lookup (local, this, inode);
 }
 
+static int32_t
+afr_discovery_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *dict,
+                   dict_t *xdata)
+{
+        int              ret            = 0;
+        char            *pathinfo       = NULL;
+        gf_boolean_t     is_local        = _gf_false;
+        afr_private_t   *priv           = NULL;
+        int32_t          child_index    = -1;
+
+        if (op_ret != 0) {
+                goto out;
+        }
+
+        ret = dict_get_str (dict, GF_XATTR_PATHINFO_KEY, &pathinfo);
+        if (ret != 0) {
+                goto out;
+        }
+
+        ret = afr_local_pathinfo (pathinfo, &is_local);
+        if (ret) {
+                goto out;
+        }
+
+        priv = this->private;
+        /*
+         * Note that one local subvolume will override another here.  The only
+         * way to avoid that would be to retain extra information about whether
+         * the previous read_child is local, and it's just not worth it.  Even
+         * the slowest local subvolume is far preferable to a remote one.
+         */
+        if (is_local) {
+                child_index = (int32_t)(long)cookie;
+                gf_log (this->name, GF_LOG_INFO,
+                        "selecting local read_child %s",
+                        priv->children[child_index]->name);
+                priv->read_child = child_index;
+        }
+
+out:
+        STACK_DESTROY(frame->root);
+        return 0;
+}
+
+static void
+afr_attempt_local_discovery (xlator_t *this, int32_t child_index)
+{
+        call_frame_t    *newframe = NULL;
+        loc_t            tmploc = {0,};
+        afr_private_t   *priv = this->private;
+
+        newframe = create_frame(this,this->ctx->pool);
+        if (!newframe) {
+                return;
+        }
+
+        tmploc.gfid[sizeof(tmploc.gfid)-1] = 1;
+        STACK_WIND_COOKIE (newframe, afr_discovery_cbk,
+                           (void *)(long)child_index,
+                           priv->children[child_index],
+                           priv->children[child_index]->fops->getxattr,
+                           &tmploc, GF_XATTR_PATHINFO_KEY, NULL);
+}
+
 static void
 afr_lookup_handle_success (afr_local_t *local, xlator_t *this, int32_t child_index,
                            int32_t op_ret, int32_t op_errno, inode_t *inode,
                            struct iatt *buf, dict_t *xattr,
                            struct iatt *postparent)
 {
+        afr_private_t   *priv   = this->private;
+
         if (local->success_count == 0) {
                 if (local->op_errno != ESTALE) {
                         local->op_ret = op_ret;
@@ -1902,6 +2321,11 @@ afr_lookup_handle_success (afr_local_t *local, xlator_t *this, int32_t child_ind
 
         afr_lookup_cache_args (local, child_index, xattr,
                                buf, postparent);
+
+        if (local->do_discovery && (priv->read_child == (-1))) {
+                afr_attempt_local_discovery(this,child_index);
+        }
+
         local->cont.lookup.success_children[local->success_count] = child_index;
         local->success_count++;
 }
@@ -1949,6 +2373,7 @@ afr_lookup_cont_init (afr_local_t *local, unsigned int child_count)
         struct iatt       *iatts         = NULL;
         int32_t           *success_children = NULL;
         int32_t           *sources       = NULL;
+        int32_t           **pending_matrix = NULL;
 
         GF_ASSERT (local);
         local->cont.lookup.xattrs = GF_CALLOC (child_count,
@@ -1981,6 +2406,11 @@ afr_lookup_cont_init (afr_local_t *local, unsigned int child_count)
                 goto out;
         local->cont.lookup.sources = sources;
 
+        pending_matrix = afr_matrix_create (child_count, child_count);
+        if (NULL == pending_matrix)
+                goto out;
+        local->cont.lookup.pending_matrix = pending_matrix;
+
         ret = 0;
 out:
         return ret;
@@ -1998,7 +2428,7 @@ afr_lookup (call_frame_t *frame, xlator_t *this,
         int               call_count     = 0;
         uint64_t          ctx            = 0;
         int32_t           op_errno       = 0;
-
+        int               allow_sh       = 0;
         priv = this->private;
 
         AFR_LOCAL_ALLOC_OR_GOTO (local, out);
@@ -2008,25 +2438,38 @@ afr_lookup (call_frame_t *frame, xlator_t *this,
         frame->local = local;
         local->fop = GF_FOP_LOOKUP;
 
-        if (!strcmp (loc->path, "/" GF_REPLICATE_TRASH_DIR)) {
-                op_errno = ENOENT;
+        loc_copy (&local->loc, loc);
+        ret = loc_path (&local->loc, NULL);
+        if (ret < 0) {
+                op_errno = EINVAL;
                 goto out;
         }
 
-        loc_copy (&local->loc, loc);
+        if (local->loc.path &&
+            (strcmp (local->loc.path, "/" GF_REPLICATE_TRASH_DIR) == 0)) {
+                op_errno = EPERM;
+                ret = -1;
+                goto out;
+        }
 
-        ret = inode_ctx_get (loc->inode, this, &ctx);
+        ret = inode_ctx_get (local->loc.inode, this, &ctx);
         if (ret == 0) {
                 /* lookup is a revalidate */
 
                 local->read_child_index = afr_inode_get_read_ctx (this,
-                                                                  loc->inode,
-                                                                  NULL);
+                                                               local->loc.inode,
+                                                               NULL);
         } else {
                 LOCK (&priv->read_child_lock);
                 {
-                        local->read_child_index = (++priv->read_child_rr)
-                                % (priv->child_count);
+                        if (priv->hash_mode) {
+                                local->read_child_index = -1;
+                        }
+                        else {
+                                local->read_child_index =
+                                        (++priv->read_child_rr) %
+                                        (priv->child_count);
+                        }
                 }
                 UNLOCK (&priv->read_child_lock);
                 local->cont.lookup.fresh_lookup = _gf_true;
@@ -2057,24 +2500,33 @@ afr_lookup (call_frame_t *frame, xlator_t *this,
         /* By default assume ENOTCONN. On success it will be set to 0. */
         local->op_errno = ENOTCONN;
 
-        local->call_count = afr_up_children_count (local->child_up,
-                                                   priv->child_count);
-        ret = afr_lookup_xattr_req_prepare (local, this, xattr_req, loc,
+        ret = dict_get_int32 (xattr_req, "allow-sh-for-running-transaction",
+                              &allow_sh);
+        dict_del (xattr_req, "allow-sh-for-running-transaction");
+        local->allow_sh_for_running_transaction = allow_sh;
+
+        ret = afr_lookup_xattr_req_prepare (local, this, xattr_req, &local->loc,
                                             &gfid_req);
         if (ret) {
                 local->op_errno = -ret;
                 goto out;
         }
         afr_lookup_save_gfid (local->cont.lookup.gfid_req, gfid_req,
-                              loc);
+                              &local->loc);
         local->fop = GF_FOP_LOOKUP;
+        if (priv->choose_local && !priv->did_discovery) {
+                if (gfid_req && __is_root_gfid(gfid_req)) {
+                        local->do_discovery = _gf_true;
+                        priv->did_discovery = _gf_true;
+                }
+        }
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND_COOKIE (frame, afr_lookup_cbk,
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->lookup,
-                                           loc, local->xattr_req);
+                                           &local->loc, local->xattr_req);
                         if (!--call_count)
                                 break;
                 }
@@ -2168,8 +2620,11 @@ __afr_fd_ctx_set (xlator_t *this, fd_t *fd)
                 goto out;
         }
 
-        INIT_LIST_HEAD (&fd_ctx->paused_calls);
+	pthread_mutex_init (&fd_ctx->delay_lock, NULL);
         INIT_LIST_HEAD (&fd_ctx->entries);
+        fd_ctx->call_child = -1;
+
+        INIT_LIST_HEAD (&fd_ctx->eager_locked);
 
         ret = __fd_ctx_set (fd, this, (uint64_t)(long) fd_ctx);
         if (ret)
@@ -2197,134 +2652,70 @@ afr_fd_ctx_set (xlator_t *this, fd_t *fd)
 /* {{{ flush */
 
 int
-afr_flush_unwind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (flush, main_frame,
-                                  local->op_ret, local->op_errno);
-        }
-
-        return 0;
-}
-
-
-int
-afr_flush_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+afr_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
         int call_count  = -1;
-        int child_index = (long) cookie;
-        int need_unwind = 0;
 
         local = frame->local;
-        priv  = this->private;
 
         LOCK (&frame->lock);
         {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
                 if (op_ret != -1) {
                         if (local->success_count == 0) {
                                 local->op_ret = op_ret;
                         }
                         local->success_count++;
-
-                        if (local->success_count == priv->wait_count) {
-                                need_unwind = 1;
-                        }
                 }
 
                 local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
-        if (need_unwind)
-                afr_flush_unwind (frame, this);
+	call_count = afr_frame_return (frame);
 
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+	if (call_count == 0)
+		AFR_STACK_UNWIND(flush, frame, local->op_ret,
+				 local->op_errno, NULL);
 
         return 0;
 }
 
-
-int
-afr_flush_wind (call_frame_t *frame, xlator_t *this)
+static int
+afr_flush_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int i = 0;
-        int call_count = -1;
+        int           i      = 0;
+        afr_local_t   *local = NULL;
+        afr_private_t *priv  = NULL;
+        int call_count       = -1;
 
-        local = frame->local;
         priv = this->private;
-
-        call_count = afr_up_children_count (local->child_up, priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
+        local = frame->local;
+        call_count = local->call_count;
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_flush_wind_cbk,
+                        STACK_WIND_COOKIE (frame, afr_flush_cbk,
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->flush,
-                                           local->fd);
-
+                                           local->fd, NULL);
                         if (!--call_count)
                                 break;
+
                 }
         }
 
         return 0;
 }
 
-
 int
-afr_flush_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
-
-
-int
-afr_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+afr_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
         afr_local_t   *local = NULL;
-        call_frame_t  *transaction_frame = NULL;
+        call_stub_t   *stub = NULL;
         int            ret        = -1;
         int            op_errno   = 0;
 
@@ -2334,47 +2725,27 @@ afr_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
 
         priv = this->private;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+	AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+	local = frame->local;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
+	ret = afr_local_init(local, priv, &op_errno);
+	if (ret < 0)
+		goto out;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
-        local->op = GF_FOP_FLUSH;
-
-        local->transaction.fop    = afr_flush_wind;
-        local->transaction.done   = afr_flush_done;
-        local->transaction.unwind = afr_flush_unwind;
-
-        local->fd                 = fd_ref (fd);
-
-        local->transaction.main_frame = frame;
-        local->transaction.start  = 0;
-        local->transaction.len    = 0;
-
-        ret = afr_open_fd_fix (transaction_frame, this, _gf_false);
-        if (ret) {
-                op_errno = -ret;
+	local->fd = fd_ref(fd);
+        stub = fop_flush_stub (frame, afr_flush_wrapper, fd, xdata);
+        if (!stub) {
+                ret = -1;
+                op_errno = ENOMEM;
                 goto out;
         }
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
 
+        afr_delayed_changelog_wake_resume (this, fd, stub);
+	ret = 0;
 
-        ret = 0;
 out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-
-                AFR_STACK_UNWIND (flush, frame, -1, op_errno);
-        }
+	if (ret < 0)
+		AFR_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -2388,8 +2759,6 @@ afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd)
         uint64_t        ctx = 0;
         afr_fd_ctx_t    *fd_ctx = NULL;
         int             ret = 0;
-        afr_fd_paused_call_t *paused_call = NULL;
-        afr_fd_paused_call_t *tmp = NULL;
 
         ret = fd_ctx_get (fd, this, &ctx);
         if (ret < 0)
@@ -2398,28 +2767,18 @@ afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd)
         fd_ctx = (afr_fd_ctx_t *)(long) ctx;
 
         if (fd_ctx) {
-                if (fd_ctx->pre_op_done)
-                        GF_FREE (fd_ctx->pre_op_done);
+                GF_FREE (fd_ctx->pre_op_done);
 
-                if (fd_ctx->opened_on)
-                        GF_FREE (fd_ctx->opened_on);
+                GF_FREE (fd_ctx->opened_on);
 
-                if (fd_ctx->locked_on)
-                        GF_FREE (fd_ctx->locked_on);
+                GF_FREE (fd_ctx->locked_on);
 
-                if (fd_ctx->pre_op_piggyback)
-                        GF_FREE (fd_ctx->pre_op_piggyback);
-                list_for_each_entry_safe (paused_call, tmp, &fd_ctx->paused_calls,
-                                          call_list) {
-                        list_del_init (&paused_call->call_list);
-                        GF_FREE (paused_call);
-                }
+                GF_FREE (fd_ctx->pre_op_piggyback);
+                GF_FREE (fd_ctx->lock_piggyback);
 
-                if (fd_ctx->lock_piggyback)
-                        GF_FREE (fd_ctx->lock_piggyback);
+                GF_FREE (fd_ctx->lock_acquired);
 
-                if (fd_ctx->lock_acquired)
-                        GF_FREE (fd_ctx->lock_acquired);
+		pthread_mutex_destroy (&fd_ctx->delay_lock);
 
                 GF_FREE (fd_ctx);
         }
@@ -2457,14 +2816,25 @@ afr_release (xlator_t *this, fd_t *fd)
 /* {{{ fsync */
 
 int
+afr_fsync_unwind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+        AFR_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
+        return 0;
+}
+
+int
 afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf)
+               struct iatt *postbuf, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         int call_count = -1;
         int child_index = (long) cookie;
         int read_child  = 0;
+	call_stub_t *stub = NULL;
 
         local = frame->local;
 
@@ -2480,13 +2850,13 @@ afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->op_ret = 0;
 
                         if (local->success_count == 0) {
-                                local->cont.fsync.prebuf  = *prebuf;
-                                local->cont.fsync.postbuf = *postbuf;
+                                local->cont.inode_wfop.prebuf  = *prebuf;
+                                local->cont.inode_wfop.postbuf = *postbuf;
                         }
 
                         if (child_index == read_child) {
-                                local->cont.fsync.prebuf  = *prebuf;
-                                local->cont.fsync.postbuf = *postbuf;
+                                local->cont.inode_wfop.prebuf  = *prebuf;
+                                local->cont.inode_wfop.postbuf = *postbuf;
                         }
 
                         local->success_count++;
@@ -2499,9 +2869,32 @@ afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
-                AFR_STACK_UNWIND (fsync, frame, local->op_ret, local->op_errno,
-                                  &local->cont.fsync.prebuf,
-                                  &local->cont.fsync.postbuf);
+		/* Make a stub out of the frame, and register it
+		   with the waking up post-op. When the call-stub resumes,
+		   we are guaranteed that there was no post-op pending
+		   (i.e changelogs were unset in the server). This is an
+		   essential "guarantee", that fsync() returns only after
+		   completely finishing EVERYTHING, including the delayed
+		   post-op. This guarantee is expected by FUSE graph switching
+		   for example.
+		*/
+		stub = fop_fsync_cbk_stub (frame, afr_fsync_unwind_cbk,
+                                           local->op_ret, local->op_errno,
+                                           &local->cont.inode_wfop.prebuf,
+                                           &local->cont.inode_wfop.postbuf,
+                                           xdata);
+		if (!stub) {
+			AFR_STACK_UNWIND (fsync, frame, -1, ENOMEM, 0, 0, 0);
+			return 0;
+		}
+
+		/* If no new unstable writes happened between the
+		   time we cleared the unstable write witness flag in afr_fsync
+		   and now, calling afr_delayed_changelog_wake_up() should
+		   wake up and skip over the fsync phase and go straight to
+		   afr_changelog_post_op_now()
+		*/
+		afr_delayed_changelog_wake_resume (this, local->fd, stub);
         }
 
         return 0;
@@ -2510,7 +2903,7 @@ afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-           int32_t datasync)
+           int32_t datasync, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local = NULL;
@@ -2536,13 +2929,17 @@ afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         local->fd             = fd_ref (fd);
 
+	if (afr_fd_has_witnessed_unstable_write (this, fd)) {
+		/* don't care. we only wanted to CLEAR the bit */
+	}
+
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND_COOKIE (frame, afr_fsync_cbk,
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->fsync,
-                                           fd, datasync);
+                                           fd, datasync, xdata);
                         if (!--call_count)
                                 break;
                 }
@@ -2551,7 +2948,7 @@ afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL);
+                AFR_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -2561,7 +2958,8 @@ out:
 
 int32_t
 afr_fsyncdir_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno)
+                  xlator_t *this, int32_t op_ret, int32_t op_errno,
+                  dict_t *xdata)
 {
         afr_local_t *local = NULL;
         int call_count = -1;
@@ -2581,7 +2979,7 @@ afr_fsyncdir_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (fsyncdir, frame, local->op_ret,
-                                  local->op_errno);
+                                  local->op_errno, xdata);
 
         return 0;
 }
@@ -2589,7 +2987,7 @@ afr_fsyncdir_cbk (call_frame_t *frame, void *cookie,
 
 int32_t
 afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              int32_t datasync)
+              int32_t datasync, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local = NULL;
@@ -2618,7 +3016,7 @@ afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
                         STACK_WIND (frame, afr_fsyncdir_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->fsyncdir,
-                                    fd, datasync);
+                                    fd, datasync, xdata);
                         if (!--call_count)
                                 break;
                 }
@@ -2627,7 +3025,7 @@ afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (fsyncdir, frame, -1, op_errno);
+                AFR_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
         return 0;
 }
 
@@ -2638,7 +3036,7 @@ out:
 int32_t
 afr_xattrop_cbk (call_frame_t *frame, void *cookie,
                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 dict_t *xattr)
+                 dict_t *xattr, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         int call_count = -1;
@@ -2661,7 +3059,7 @@ afr_xattrop_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (xattrop, frame, local->op_ret, local->op_errno,
-                                  local->cont.xattrop.xattr);
+                local->cont.xattrop.xattr, xdata);
 
         return 0;
 }
@@ -2669,7 +3067,7 @@ afr_xattrop_cbk (call_frame_t *frame, void *cookie,
 
 int32_t
 afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t optype, dict_t *xattr)
+             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
@@ -2698,7 +3096,7 @@ afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
                         STACK_WIND (frame, afr_xattrop_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->xattrop,
-                                    loc, optype, xattr);
+                                    loc, optype, xattr, xdata);
                         if (!--call_count)
                                 break;
                 }
@@ -2707,7 +3105,7 @@ afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL);
+                AFR_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
@@ -2718,7 +3116,7 @@ out:
 int32_t
 afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
                   xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  dict_t *xattr)
+                  dict_t *xattr, dict_t *xdata)
 {
         afr_local_t *local = NULL;
 
@@ -2743,7 +3141,7 @@ afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (fxattrop, frame, local->op_ret, local->op_errno,
-                                  local->cont.fxattrop.xattr);
+                                  local->cont.fxattrop.xattr, xdata);
 
         return 0;
 }
@@ -2751,7 +3149,7 @@ afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
 
 int32_t
 afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              gf_xattrop_flags_t optype, dict_t *xattr)
+              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
@@ -2780,7 +3178,7 @@ afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
                         STACK_WIND (frame, afr_fxattrop_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->fxattrop,
-                                    fd, optype, xattr);
+                                    fd, optype, xattr, xdata);
                         if (!--call_count)
                                 break;
                 }
@@ -2789,7 +3187,7 @@ afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL);
+                AFR_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
@@ -2798,7 +3196,7 @@ out:
 
 int32_t
 afr_inodelk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno)
+                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
         afr_local_t *local = NULL;
@@ -2819,7 +3217,7 @@ afr_inodelk_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (inodelk, frame, local->op_ret,
-                                  local->op_errno);
+                                  local->op_errno, xdata);
 
         return 0;
 }
@@ -2827,7 +3225,8 @@ afr_inodelk_cbk (call_frame_t *frame, void *cookie,
 
 int32_t
 afr_inodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock)
+             const char *volume, loc_t *loc, int32_t cmd,
+             struct gf_flock *flock, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
@@ -2856,7 +3255,7 @@ afr_inodelk (call_frame_t *frame, xlator_t *this,
                         STACK_WIND (frame, afr_inodelk_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->inodelk,
-                                    volume, loc, cmd, flock);
+                                    volume, loc, cmd, flock, xdata);
 
                         if (!--call_count)
                                 break;
@@ -2866,14 +3265,15 @@ afr_inodelk (call_frame_t *frame, xlator_t *this,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (inodelk, frame, -1, op_errno);
+                AFR_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
         return 0;
 }
 
 
 int32_t
 afr_finodelk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno)
+                  xlator_t *this, int32_t op_ret, int32_t op_errno,
+                  dict_t *xdata)
 
 {
         afr_local_t *local = NULL;
@@ -2894,7 +3294,7 @@ afr_finodelk_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (finodelk, frame, local->op_ret,
-                                  local->op_errno);
+                                  local->op_errno, xdata);
 
         return 0;
 }
@@ -2902,7 +3302,8 @@ afr_finodelk_cbk (call_frame_t *frame, void *cookie,
 
 int32_t
 afr_finodelk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock)
+              const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock,
+              dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
@@ -2931,7 +3332,7 @@ afr_finodelk (call_frame_t *frame, xlator_t *this,
                         STACK_WIND (frame, afr_finodelk_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->finodelk,
-                                    volume, fd, cmd, flock);
+                                    volume, fd, cmd, flock, xdata);
 
                         if (!--call_count)
                                 break;
@@ -2941,15 +3342,14 @@ afr_finodelk (call_frame_t *frame, xlator_t *this,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (finodelk, frame, -1, op_errno);
+                AFR_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
         return 0;
 }
 
 
 int32_t
-afr_entrylk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno)
-
+afr_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         int call_count = -1;
@@ -2969,7 +3369,7 @@ afr_entrylk_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (entrylk, frame, local->op_ret,
-                                  local->op_errno);
+                                  local->op_errno, xdata);
 
         return 0;
 }
@@ -2978,7 +3378,8 @@ afr_entrylk_cbk (call_frame_t *frame, void *cookie,
 int32_t
 afr_entrylk (call_frame_t *frame, xlator_t *this,
              const char *volume, loc_t *loc,
-             const char *basename, entrylk_cmd cmd, entrylk_type type)
+             const char *basename, entrylk_cmd cmd, entrylk_type type,
+             dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
@@ -3007,7 +3408,7 @@ afr_entrylk (call_frame_t *frame, xlator_t *this,
                         STACK_WIND (frame, afr_entrylk_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->entrylk,
-                                    volume, loc, basename, cmd, type);
+                                    volume, loc, basename, cmd, type, xdata);
 
                         if (!--call_count)
                                 break;
@@ -3017,7 +3418,7 @@ afr_entrylk (call_frame_t *frame, xlator_t *this,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (entrylk, frame, -1, op_errno);
+                AFR_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
         return 0;
 }
 
@@ -3025,7 +3426,7 @@ out:
 
 int32_t
 afr_fentrylk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno)
+                  xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
         afr_local_t *local = NULL;
@@ -3046,7 +3447,7 @@ afr_fentrylk_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (fentrylk, frame, local->op_ret,
-                                  local->op_errno);
+                                  local->op_errno, xdata);
 
         return 0;
 }
@@ -3055,7 +3456,8 @@ afr_fentrylk_cbk (call_frame_t *frame, void *cookie,
 int32_t
 afr_fentrylk (call_frame_t *frame, xlator_t *this,
               const char *volume, fd_t *fd,
-              const char *basename, entrylk_cmd cmd, entrylk_type type)
+              const char *basename, entrylk_cmd cmd,
+              entrylk_type type, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
@@ -3084,7 +3486,7 @@ afr_fentrylk (call_frame_t *frame, xlator_t *this,
                         STACK_WIND (frame, afr_fentrylk_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->fentrylk,
-                                    volume, fd, basename, cmd, type);
+                                    volume, fd, basename, cmd, type, xdata);
 
                         if (!--call_count)
                                 break;
@@ -3094,14 +3496,14 @@ afr_fentrylk (call_frame_t *frame, xlator_t *this,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (fentrylk, frame, -1, op_errno);
+                AFR_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
         return 0;
 }
 
 int32_t
 afr_statfs_cbk (call_frame_t *frame, void *cookie,
                 xlator_t *this, int32_t op_ret, int32_t op_errno,
-                struct statvfs *statvfs)
+                struct statvfs *statvfs, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         int call_count = 0;
@@ -3132,7 +3534,7 @@ afr_statfs_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
-                                  &local->cont.statfs.buf);
+                                  &local->cont.statfs.buf, xdata);
 
         return 0;
 }
@@ -3140,7 +3542,7 @@ afr_statfs_cbk (call_frame_t *frame, void *cookie,
 
 int32_t
 afr_statfs (call_frame_t *frame, xlator_t *this,
-            loc_t *loc)
+            loc_t *loc, dict_t *xdata)
 {
         afr_private_t *  priv        = NULL;
         int              child_count = 0;
@@ -3171,7 +3573,7 @@ afr_statfs (call_frame_t *frame, xlator_t *this,
                         STACK_WIND (frame, afr_statfs_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->statfs,
-                                    loc);
+                                    loc, xdata);
                         if (!--call_count)
                                 break;
                 }
@@ -3180,14 +3582,15 @@ afr_statfs (call_frame_t *frame, xlator_t *this,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (statfs, frame, -1, op_errno, NULL);
+                AFR_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 afr_lk_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+                   int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                   dict_t *xdata)
 {
         afr_local_t * local = NULL;
         int call_count = -1;
@@ -3197,7 +3600,7 @@ afr_lk_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  lock);
+                                  lock, xdata);
 
         return 0;
 }
@@ -3219,7 +3622,7 @@ afr_lk_unlock (call_frame_t *frame, xlator_t *this)
 
         if (call_count == 0) {
                 AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  &local->cont.lk.ret_flock);
+                                  &local->cont.lk.ret_flock, NULL);
                 return 0;
         }
 
@@ -3233,7 +3636,7 @@ afr_lk_unlock (call_frame_t *frame, xlator_t *this)
                                     priv->children[i],
                                     priv->children[i]->fops->lk,
                                     local->fd, F_SETLK,
-                                    &local->cont.lk.user_flock);
+                                    &local->cont.lk.user_flock, NULL);
 
                         if (!--call_count)
                                 break;
@@ -3246,7 +3649,7 @@ afr_lk_unlock (call_frame_t *frame, xlator_t *this)
 
 int32_t
 afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+            int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
@@ -3281,12 +3684,12 @@ afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                    priv->children[child_index],
                                    priv->children[child_index]->fops->lk,
                                    local->fd, local->cont.lk.cmd,
-                                   &local->cont.lk.user_flock);
+                                   &local->cont.lk.user_flock, xdata);
         } else if (local->op_ret == -1) {
                 /* all nodes have gone down */
 
                 AFR_STACK_UNWIND (lk, frame, -1, ENOTCONN,
-                                  &local->cont.lk.ret_flock);
+                                  &local->cont.lk.ret_flock, NULL);
         } else {
                 /* locking has succeeded on all nodes that are up */
 
@@ -3304,7 +3707,7 @@ afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 */
                 AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  &local->cont.lk.ret_flock);
+                                  &local->cont.lk.ret_flock, NULL);
         }
 
         return 0;
@@ -3313,7 +3716,7 @@ afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 afr_lk (call_frame_t *frame, xlator_t *this,
-        fd_t *fd, int32_t cmd, struct gf_flock *flock)
+        fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local = NULL;
@@ -3351,12 +3754,12 @@ afr_lk (call_frame_t *frame, xlator_t *this,
         STACK_WIND_COOKIE (frame, afr_lk_cbk, (void *) (long) 0,
                            priv->children[i],
                            priv->children[i]->fops->lk,
-                           fd, cmd, flock);
+                           fd, cmd, flock, xdata);
 
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (lk, frame, -1, op_errno, NULL);
+                AFR_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
@@ -3372,8 +3775,7 @@ afr_forget (xlator_t *this, inode_t *inode)
                 goto out;
 
         ctx = (afr_inode_ctx_t *)(long)ctx_addr;
-        if (ctx->fresh_children)
-                GF_FREE (ctx->fresh_children);
+        GF_FREE (ctx->fresh_children);
         GF_FREE (ctx);
 out:
         return 0;
@@ -3461,6 +3863,14 @@ afr_notify (xlator_t *this, int32_t event,
         if (!priv)
                 return 0;
 
+        /*
+         * We need to reset this in case children come up in "staggered"
+         * fashion, so that we discover a late-arriving local subvolume.  Note
+         * that we could end up issuing N lookups to the first subvolume, and
+         * O(N^2) overall, but N is small for AFR so it shouldn't be an issue.
+         */
+        priv->did_discovery = _gf_false;
+
         had_heard_from_all = 1;
         for (i = 0; i < priv->child_count; i++) {
                 if (!priv->last_event[i]) {
@@ -3680,6 +4090,17 @@ afr_local_init (afr_local_t *local, afr_private_t *priv, int32_t *op_errno)
                 goto out;
         }
 
+        local->transaction.postop_piggybacked = GF_CALLOC (priv->child_count,
+							   sizeof (int),
+							   gf_afr_mt_int32_t);
+        if (!local->transaction.postop_piggybacked) {
+                if (op_errno)
+                        *op_errno = ENOMEM;
+                goto out;
+        }
+
+	local->append_write = _gf_false;
+
         ret = 0;
 out:
         return ret;
@@ -3691,16 +4112,6 @@ afr_internal_lock_init (afr_internal_lock_t *lk, size_t child_count,
 {
         int             ret = -ENOMEM;
 
-        lk->inode_locked_nodes = GF_CALLOC (sizeof (*lk->inode_locked_nodes),
-                                            child_count, gf_afr_mt_char);
-        if (NULL == lk->inode_locked_nodes)
-                goto out;
-
-        lk->entry_locked_nodes = GF_CALLOC (sizeof (*lk->entry_locked_nodes),
-                                            child_count, gf_afr_mt_char);
-        if (NULL == lk->entry_locked_nodes)
-                goto out;
-
         lk->locked_nodes = GF_CALLOC (sizeof (*lk->locked_nodes),
                                       child_count, gf_afr_mt_char);
         if (NULL == lk->locked_nodes)
@@ -3759,6 +4170,21 @@ out:
 }
 
 int
+afr_inodelk_init (afr_inodelk_t *lk, char *dom, size_t child_count)
+{
+        int             ret = -ENOMEM;
+
+        lk->domain = dom;
+        lk->locked_nodes = GF_CALLOC (sizeof (*lk->locked_nodes),
+                                      child_count, gf_afr_mt_char);
+        if (NULL == lk->locked_nodes)
+                goto out;
+        ret = 0;
+out:
+        return ret;
+}
+
+int
 afr_transaction_local_init (afr_local_t *local, xlator_t *this)
 {
         int            child_up_count = 0;
@@ -3771,6 +4197,14 @@ afr_transaction_local_init (afr_local_t *local, xlator_t *this)
         if (ret < 0)
                 goto out;
 
+        if ((local->transaction.type == AFR_DATA_TRANSACTION) ||
+            (local->transaction.type == AFR_METADATA_TRANSACTION)) {
+                ret = afr_inodelk_init (&local->internal_lock.inodelk[0],
+                                        this->name, priv->child_count);
+                if (ret < 0)
+                        goto out;
+        }
+
         ret = -ENOMEM;
         child_up_count = afr_up_children_count (local->child_up,
                                                 priv->child_count);
@@ -3792,14 +4226,6 @@ afr_transaction_local_init (afr_local_t *local, xlator_t *this)
         if (!local->fresh_children)
                 goto out;
 
-        if (local->fd) {
-                local->fd_open_on = GF_CALLOC (sizeof (*local->fd_open_on),
-                                               priv->child_count,
-                                               gf_afr_mt_char);
-                if (!local->fd_open_on)
-                        goto out;
-        }
-
         local->transaction.pre_op = GF_CALLOC (sizeof (*local->transaction.pre_op),
                                                priv->child_count,
                                                gf_afr_mt_char);
@@ -3811,11 +4237,12 @@ afr_transaction_local_init (afr_local_t *local, xlator_t *this)
         if (!local->pending)
                 goto out;
 
-        local->transaction.child_errno =
-                GF_CALLOC (sizeof (*local->transaction.child_errno),
-                           priv->child_count,
-                           gf_afr_mt_int32_t);
-        local->transaction.erase_pending = 1;
+        local->transaction.txn_changelog = afr_matrix_create (priv->child_count,
+                                                           AFR_NUM_CHANGE_LOGS);
+        if (!local->transaction.txn_changelog)
+                goto out;
+
+	INIT_LIST_HEAD (&local->transaction.eager_locked);
 
         ret = 0;
 out:
@@ -3911,7 +4338,7 @@ afr_set_low_priority (call_frame_t *frame)
 
 int
 afr_child_fd_ctx_set (xlator_t *this, fd_t *fd, int32_t child,
-                      int flags, int32_t wbflags)
+                      int flags)
 {
         int             ret = 0;
         uint64_t        ctx = 0;
@@ -3936,7 +4363,6 @@ afr_child_fd_ctx_set (xlator_t *this, fd_t *fd, int32_t child,
         fd_ctx->opened_on[child] = AFR_FD_OPENED;
         if (!IA_ISDIR (fd->inode->ia_type)) {
                 fd_ctx->flags            = flags;
-                fd_ctx->wbflags          = wbflags;
         }
         ret = 0;
 out:
@@ -3991,7 +4417,6 @@ afr_priv_destroy (afr_private_t *priv)
         GF_FREE (priv->shd.pos);
         GF_FREE (priv->shd.pending);
         GF_FREE (priv->shd.inprogress);
-        GF_FREE (priv->shd.sh_times);
 //        for (i = 0; i < priv->child_count; i++)
 //                if (priv->shd.timer && priv->shd.timer[i])
 //                        gf_timer_call_cancel (this->ctx, priv->shd.timer[i]);
@@ -4006,6 +4431,16 @@ afr_priv_destroy (afr_private_t *priv)
         if (priv->shd.split_brain)
                 eh_destroy (priv->shd.split_brain);
 
+        for (i = 0; i < priv->child_count; i++)
+        {
+                if (priv->shd.statistics[i])
+                        eh_destroy (priv->shd.statistics[i]);
+        }
+
+        GF_FREE (priv->shd.statistics);
+
+        GF_FREE (priv->shd.crawl_events);
+
         GF_FREE (priv->last_event);
         if (priv->pending_key) {
                 for (i = 0; i < priv->child_count; i++)
@@ -4032,3 +4467,125 @@ xlator_subvolume_count (xlator_t *this)
                 i++;
         return i;
 }
+
+inline gf_boolean_t
+afr_is_errno_set (int *child_errno, int child)
+{
+        return child_errno[child];
+}
+
+inline gf_boolean_t
+afr_is_errno_unset (int *child_errno, int child)
+{
+        return !afr_is_errno_set (child_errno, child);
+}
+
+void
+afr_prepare_new_entry_pending_matrix (int32_t **pending,
+                                      gf_boolean_t (*is_pending) (int *, int),
+                                      int *ctx, struct iatt *buf,
+                                      unsigned int child_count)
+{
+        int midx = 0;
+        int idx  = 0;
+        int i    = 0;
+
+        midx = afr_index_for_transaction_type (AFR_METADATA_TRANSACTION);
+        if (IA_ISDIR (buf->ia_type))
+                idx = afr_index_for_transaction_type (AFR_ENTRY_TRANSACTION);
+        else if (IA_ISREG (buf->ia_type))
+                idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
+        else
+                idx = -1;
+        for (i = 0; i < child_count; i++) {
+                if (is_pending (ctx, i)) {
+                        pending[i][midx] = hton32 (1);
+                        if (idx == -1)
+                                continue;
+                        pending[i][idx] = hton32 (1);
+                }
+        }
+}
+
+gf_boolean_t
+afr_is_fd_fixable (fd_t *fd)
+{
+        if (!fd || !fd->inode)
+                return _gf_false;
+        else if (fd_is_anonymous (fd))
+                return _gf_false;
+        else if (uuid_is_null (fd->inode->gfid))
+                return _gf_false;
+
+        return _gf_true;
+}
+
+void
+afr_handle_open_fd_count (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        inode_t         *inode = NULL;
+        afr_inode_ctx_t *ctx   = NULL;
+
+        local = frame->local;
+
+        if (local->fd)
+                inode = local->fd->inode;
+        else
+                inode = local->loc.inode;
+
+        if (!inode)
+                return;
+
+        LOCK (&inode->lock);
+        {
+                ctx = __afr_inode_ctx_get (inode, this);
+                ctx->open_fd_count = local->open_fd_count;
+        }
+        UNLOCK (&inode->lock);
+}
+
+int
+afr_initialise_statistics (xlator_t *this)
+{
+        afr_private_t       *priv = NULL;
+        int                 ret = -1;
+        int                 i = 0;
+        int                 child_count = 0;
+        eh_t                *stats_per_brick = NULL;
+        shd_crawl_event_t   ***shd_crawl_events = NULL;
+        priv = this->private;
+
+        priv->shd.statistics = GF_CALLOC (sizeof(eh_t *), priv->child_count,
+                                          gf_common_mt_eh_t);
+        if (!priv->shd.statistics) {
+                ret = -1;
+                goto out;
+        }
+        child_count = priv->child_count;
+        for (i=0; i < child_count ; i++) {
+                stats_per_brick = eh_new (AFR_STATISTICS_HISTORY_SIZE,
+                                          _gf_false,
+                                          _destroy_crawl_event_data);
+                if (!stats_per_brick) {
+                        ret = -1;
+                        goto out;
+                }
+                priv->shd.statistics[i] = stats_per_brick;
+
+        }
+
+        shd_crawl_events = (shd_crawl_event_t***)(&priv->shd.crawl_events);
+        *shd_crawl_events  = GF_CALLOC (sizeof(shd_crawl_event_t*),
+                                        priv->child_count,
+                                        gf_afr_mt_shd_crawl_event_t);
+
+        if (!priv->shd.crawl_events) {
+                ret = -1;
+                goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+
+}
diff --git a/xlators/cluster/afr/src/afr-dir-read.c b/xlators/cluster/afr/src/afr-dir-read.c
index ee9c5d8cc..689dd84e6 100644
--- a/xlators/cluster/afr/src/afr-dir-read.c
+++ b/xlators/cluster/afr/src/afr-dir-read.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -51,7 +42,7 @@
 
 int
 afr_examine_dir_sh_unwind (call_frame_t *frame, xlator_t *this, int32_t op_ret,
-                           int32_t op_errno)
+                           int32_t op_errno, int32_t sh_failed)
 {
         afr_local_t *local  = NULL;
 
@@ -60,7 +51,7 @@ afr_examine_dir_sh_unwind (call_frame_t *frame, xlator_t *this, int32_t op_ret,
         afr_set_opendir_done (this, local->fd->inode);
 
         AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                          local->op_errno, local->fd);
+                          local->op_errno, local->fd, NULL);
 
         return 0;
 }
@@ -99,7 +90,7 @@ __checksums_differ (uint32_t *checksum, int child_count,
 int32_t
 afr_examine_dir_readdir_cbk (call_frame_t *frame, void *cookie,
                              xlator_t *this, int32_t op_ret, int32_t op_errno,
-                             gf_dirent_t *entries)
+                             gf_dirent_t *entries, dict_t *xdata)
 {
         afr_private_t *   priv        = NULL;
         afr_local_t *     local       = NULL;
@@ -137,7 +128,7 @@ afr_examine_dir_readdir_cbk (call_frame_t *frame, void *cookie,
         }
 
         list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                entry_cksum = gf_rsync_weak_checksum (entry->d_name,
+                entry_cksum = gf_rsync_weak_checksum ((unsigned char *)entry->d_name,
                                                       strlen (entry->d_name));
                 local->cont.opendir.checksum[child_index] ^= entry_cksum;
         }
@@ -152,7 +143,7 @@ afr_examine_dir_readdir_cbk (call_frame_t *frame, void *cookie,
                            (void *) (long) child_index,
                            priv->children[child_index],
                            priv->children[child_index]->fops->readdir,
-                           local->fd, 131072, last_offset);
+                           local->fd, 131072, last_offset, NULL);
 
         return 0;
 
@@ -175,7 +166,7 @@ out:
                         afr_set_opendir_done (this, inode);
 
                         AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                                          local->op_errno, local->fd);
+                                          local->op_errno, local->fd, NULL);
                 }
         }
 
@@ -208,7 +199,7 @@ afr_examine_dir (call_frame_t *frame, xlator_t *this)
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->readdir,
-                                           local->fd, 131072, 0);
+                                           local->fd, 131072, 0, NULL);
 
                         if (!--call_count)
                                 break;
@@ -222,7 +213,7 @@ afr_examine_dir (call_frame_t *frame, xlator_t *this)
 int32_t
 afr_opendir_cbk (call_frame_t *frame, void *cookie,
                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 fd_t *fd)
+                 fd_t *fd, dict_t *xdata)
 {
         afr_private_t *priv              = NULL;
         afr_local_t   *local             = NULL;
@@ -242,8 +233,7 @@ afr_opendir_cbk (call_frame_t *frame, void *cookie,
         {
                 if (op_ret >= 0) {
                         local->op_ret = op_ret;
-                        ret = afr_child_fd_ctx_set (this, fd, child_index,
-                                                    0, 0);
+                        ret = afr_child_fd_ctx_set (this, fd, child_index, 0);
                         if (ret) {
                                 local->op_ret = -1;
                                 local->op_errno = -ret;
@@ -263,7 +253,7 @@ unlock:
                         goto out;
 
                 if (!afr_is_opendir_done (this, local->fd->inode) &&
-                    up_children_count > 1) {
+                    up_children_count > 1 && priv->entry_self_heal) {
 
                         /*
                          * This is the first opendir on this inode. We need
@@ -291,7 +281,7 @@ unlock:
 
 out:
         AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                          local->op_errno, local->fd);
+                          local->op_errno, local->fd, NULL);
 
         return 0;
 }
@@ -336,7 +326,7 @@ afr_opendir (call_frame_t *frame, xlator_t *this,
                                            (void*) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->opendir,
-                                           loc, fd);
+                                           loc, fd, NULL);
 
                         if (!--call_count)
                                 break;
@@ -346,7 +336,7 @@ afr_opendir (call_frame_t *frame, xlator_t *this,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (opendir, frame, -1, op_errno, fd);
+                AFR_STACK_UNWIND (opendir, frame, -1, op_errno, fd, NULL);
 
         return 0;
 }
@@ -368,85 +358,6 @@ struct entry_name {
         struct list_head list;
 };
 
-
-static gf_boolean_t
-remembered_name (const char *name, struct list_head *entries)
-{
-        struct entry_name *e   = NULL;
-        gf_boolean_t       ret = _gf_false;
-
-        list_for_each_entry (e, entries, list) {
-                if (!strcmp (name, e->name)) {
-                        ret = _gf_true;
-                        goto out;
-                }
-        }
-
-out:
-        return ret;
-}
-
-
-static void
-afr_remember_entries (gf_dirent_t *entries, fd_t *fd)
-{
-        struct entry_name *n      = NULL;
-        gf_dirent_t       *entry  = NULL;
-        int                ret    = 0;
-        uint64_t           ctx    = 0;
-        afr_fd_ctx_t      *fd_ctx = NULL;
-
-        ret = fd_ctx_get (fd, THIS, &ctx);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "could not get fd ctx for fd=%p", fd);
-                return;
-        }
-
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-        list_for_each_entry (entry, &entries->list, list) {
-                n = GF_CALLOC (1, sizeof (*n), gf_afr_mt_entry_name);
-                n->name = gf_strdup (entry->d_name);
-                INIT_LIST_HEAD (&n->list);
-
-                list_add (&n->list, &fd_ctx->entries);
-        }
-}
-
-
-static off_t
-afr_filter_entries (gf_dirent_t *entries, fd_t *fd)
-{
-        gf_dirent_t  *entry  = NULL;
-        gf_dirent_t  *tmp    = NULL;
-        int           ret    = 0;
-        uint64_t      ctx    = 0;
-        afr_fd_ctx_t *fd_ctx = NULL;
-        off_t         offset = 0;
-
-        ret = fd_ctx_get (fd, THIS, &ctx);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "could not get fd ctx for fd=%p", fd);
-                return -1;
-        }
-
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                offset = entry->d_off;
-
-                if (remembered_name (entry->d_name, &fd_ctx->entries)) {
-                        list_del (&entry->list);
-                        GF_FREE (entry);
-                }
-        }
-
-        return offset;
-}
-
-
 static void
 afr_forget_entries (fd_t *fd)
 {
@@ -472,159 +383,55 @@ afr_forget_entries (fd_t *fd)
         }
 }
 
-
-int32_t
-afr_readdir_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 gf_dirent_t *entries)
+static void
+afr_readdir_filter_trash_dir (gf_dirent_t *entries, fd_t *fd)
 {
-        afr_local_t *   local       = NULL;
         gf_dirent_t *   entry       = NULL;
         gf_dirent_t *   tmp         = NULL;
 
-        local = frame->local;
-
-        if (op_ret == -1)
-                goto out;
-
         list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                if ((local->fd->inode == local->fd->inode->table->root)
-                    && !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
+                if (__is_root_gfid (fd->inode->gfid) &&
+                    !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
                         list_del_init (&entry->list);
                         GF_FREE (entry);
                 }
         }
-
-out:
-        AFR_STACK_UNWIND (readdir, frame, op_ret, op_errno, entries);
-
-        return 0;
 }
 
-
 int32_t
-afr_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+afr_readdir_cbk (call_frame_t *frame, void *cookie,
+                 xlator_t *this, int32_t op_ret, int32_t op_errno,
+                 gf_dirent_t *entries, dict_t *xdata)
 {
-        afr_private_t *  priv            = NULL;
-        afr_local_t *    local           = NULL;
-        xlator_t **      children        = NULL;
-        int32_t          next_call_child = -1;
-        int              ret             = 0;
-        gf_dirent_t *    entry           = NULL;
-        gf_dirent_t *    tmp             = NULL;
-        int32_t          *last_index     = NULL;
-        int32_t          read_child      = -1;
-        int32_t         *fresh_children   = NULL;
-        uint64_t         ctx             = 0;
-        afr_fd_ctx_t    *fd_ctx          = NULL;
-        off_t            offset          = 0;
-        int32_t         call_child       = -1;
+        afr_local_t     *local = NULL;
 
-        priv     = this->private;
-        children = priv->children;
+        if (op_ret == -1)
+                goto out;
 
         local = frame->local;
+        afr_readdir_filter_trash_dir (entries, local->fd);
 
-        read_child = (long) cookie;
-        last_index = &local->cont.readdir.last_index;
-        fresh_children = local->fresh_children;
-
-        /* the value of the last_index changes if afr_next_call_child is
-         * called. So to find the call_child of this callback use last_index
-         * before the next_call_child call.
-         */
-        if (*last_index == -1)
-                call_child = read_child;
-        else
-                call_child = fresh_children[*last_index];
-
-        if (priv->strict_readdir) {
-                ret = fd_ctx_get (local->fd, this, &ctx);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "could not get fd ctx for fd=%p", local->fd);
-                        op_ret   = -1;
-                        op_errno = -ret;
-                        goto out;
-                }
-
-                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-                if (op_ret == -1) {
-                        next_call_child = afr_next_call_child (fresh_children,
-                                                               local->child_up,
-                                                               priv->child_count,
-                                                               last_index,
-                                                               read_child);
-                        if (next_call_child < 0)
-                                goto out;
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "starting readdir afresh on child %d, offset %"PRId64,
-                                next_call_child, (uint64_t) 0);
-
-                        fd_ctx->failed_over = _gf_true;
-
-                        STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
-                                           (void *) (long) read_child,
-                                           children[next_call_child],
-                                           children[next_call_child]->fops->readdirp,
-                                           local->fd,
-                                           local->cont.readdir.size, 0,
-                                           local->cont.readdir.dict);
-                        return 0;
-                }
-        }
-
-        if (op_ret != -1) {
-                list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                        if ((local->fd->inode == local->fd->inode->table->root)
-                            && !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
-                                list_del_init (&entry->list);
-                                GF_FREE (entry);
-                        }
-                }
-        }
-
-        if (priv->strict_readdir) {
-                if (fd_ctx->failed_over) {
-                        if (list_empty (&entries->list)) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "no entries found");
-                                goto out;
-                        }
-
-                        offset = afr_filter_entries (entries, local->fd);
+out:
+        AFR_STACK_UNWIND (readdir, frame, op_ret, op_errno, entries, NULL);
+        return 0;
+}
 
-                        afr_remember_entries (entries, local->fd);
 
-                        if (list_empty (&entries->list)) {
-                                /* All the entries we got were duplicate. We
-                                   shouldn't send an empty list now, because
-                                   that will make the application stop reading. So
-                                   try to get more entries */
+int32_t
+afr_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                  dict_t *xdata)
+{
+        afr_local_t     *local = NULL;
 
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "trying to fetch non-duplicate entries "
-                                        "from offset %"PRId64", child %s",
-                                        offset, children[call_child]->name);
+        if (op_ret == -1)
+                goto out;
 
-                                STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
-                                                   (void *) (long) read_child,
-                                                   children[call_child],
-                                                   children[call_child]->fops->readdirp,
-                                                   local->fd, local->cont.readdir.size, offset,
-                                                   local->cont.readdir.dict);
-                                return 0;
-                        }
-                } else {
-                        afr_remember_entries (entries, local->fd);
-                }
-        }
+        local = frame->local;
+        afr_readdir_filter_trash_dir (entries, local->fd);
 
 out:
-        AFR_STACK_UNWIND (readdirp, frame, op_ret, op_errno, entries);
-
+        AFR_STACK_UNWIND (readdirp, frame, op_ret, op_errno, entries, NULL);
         return 0;
 }
 
@@ -632,15 +439,14 @@ int32_t
 afr_do_readdir (call_frame_t *frame, xlator_t *this,
                 fd_t *fd, size_t size, off_t offset, int whichop, dict_t *dict)
 {
-        afr_private_t *  priv       = NULL;
-        xlator_t **      children   = NULL;
-        int              call_child = 0;
-        afr_local_t     *local      = NULL;
-        uint64_t         ctx        = 0;
-        afr_fd_ctx_t    *fd_ctx     = NULL;
-        int              ret        = -1;
-        int32_t          op_errno   = 0;
-        uint64_t         read_child = 0;
+        afr_private_t *priv      = NULL;
+        xlator_t      **children = NULL;
+        int           call_child = 0;
+        afr_local_t   *local     = NULL;
+        afr_fd_ctx_t  *fd_ctx    = NULL;
+        int           ret        = -1;
+        int32_t       op_errno   = 0;
+        uint64_t      read_child = 0;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -665,48 +471,38 @@ afr_do_readdir (call_frame_t *frame, xlator_t *this,
         read_child = afr_inode_get_read_ctx (this, fd->inode,
                                              local->fresh_children);
         ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.readdir.last_index);
+                                  local->fresh_children,
+                                  &call_child,
+                                  &local->cont.readdir.last_index);
         if (ret < 0) {
                 op_errno = -ret;
                 goto out;
         }
 
+        fd_ctx  = afr_fd_ctx_get (fd, this);
+        if (!fd_ctx) {
+                op_errno = EBADF;
+                goto out;
+        }
+
+        if ((offset == 0) || (fd_ctx->call_child == -1)) {
+                fd_ctx->call_child = call_child;
+        } else if ((priv->readdir_failover == _gf_false) &&
+                   (call_child != fd_ctx->call_child)) {
+                op_errno = EBADF;
+                goto out;
+        }
+
         local->fd                  = fd_ref (fd);
         local->cont.readdir.size   = size;
         local->cont.readdir.dict   = (dict)? dict_ref (dict) : NULL;
 
-        if (priv->strict_readdir) {
-                ret = fd_ctx_get (fd, this, &ctx);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "could not get fd ctx for fd=%p", fd);
-                        op_errno = -ret;
-                        goto out;
-                }
-
-                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-                if (fd_ctx->last_tried != call_child) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "first up child has changed from %d to %d, "
-                                "restarting readdir from offset 0",
-                                fd_ctx->last_tried, call_child);
-
-                        fd_ctx->failed_over = _gf_true;
-                        offset = 0;
-                }
-
-                fd_ctx->last_tried = call_child;
-        }
-
         if (whichop == GF_FOP_READDIR)
                 STACK_WIND_COOKIE (frame, afr_readdir_cbk,
                                    (void *) (long) call_child,
                                    children[call_child],
                                    children[call_child]->fops->readdir, fd,
-                                   size, offset);
+                                   size, offset, dict);
         else
                 STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
                                    (void *) (long) call_child,
@@ -714,19 +510,18 @@ afr_do_readdir (call_frame_t *frame, xlator_t *this,
                                    children[call_child]->fops->readdirp, fd,
                                    size, offset, dict);
 
-        ret = 0;
+        return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (readdir, frame, -1, op_errno, NULL);
+        AFR_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 afr_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t offset)
+             off_t offset, dict_t *xdata)
 {
-        afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIR, NULL);
+        afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIR, xdata);
         return 0;
 }
 
diff --git a/xlators/cluster/afr/src/afr-dir-read.h b/xlators/cluster/afr/src/afr-dir-read.h
index 7e50a1c8c..09456d159 100644
--- a/xlators/cluster/afr/src/afr-dir-read.h
+++ b/xlators/cluster/afr/src/afr-dir-read.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __DIR_READ_H__
@@ -23,14 +14,14 @@
 
 int32_t
 afr_opendir (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, fd_t *fd);
+	     loc_t *loc, fd_t *fd, dict_t *xdata);
 
 int32_t
 afr_releasedir (xlator_t *this, fd_t *fd);
 
 int32_t
 afr_readdir (call_frame_t *frame, xlator_t *this,
-	     fd_t *fd, size_t size, off_t offset);
+	     fd_t *fd, size_t size, off_t offset, dict_t *xdata);
 
 
 int32_t
@@ -39,7 +30,7 @@ afr_readdirp (call_frame_t *frame, xlator_t *this,
 
 int32_t
 afr_checksum (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, int32_t flags);
+	      loc_t *loc, int32_t flags, dict_t *xdata);
 
 
 #endif /* __DIR_READ_H__ */
diff --git a/xlators/cluster/afr/src/afr-dir-write.c b/xlators/cluster/afr/src/afr-dir-write.c
index 1cd447e75..1943b719b 100644
--- a/xlators/cluster/afr/src/afr-dir-write.c
+++ b/xlators/cluster/afr/src/afr-dir-write.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -65,15 +56,206 @@ afr_build_parent_loc (loc_t *parent, loc_t *child, int32_t *op_errno)
                         *op_errno = ENOMEM;
                 goto out;
         }
-        parent->path = dirname (child_path);
+        parent->path = gf_strdup( dirname (child_path) );
+	if (!parent->path) {
+                if (op_errno)
+                        *op_errno = ENOMEM;
+                goto out;
+        }
         parent->inode  = inode_ref (child->parent);
         uuid_copy (parent->gfid, child->pargfid);
 
         ret = 0;
 out:
+	GF_FREE(child_path);
+
         return ret;
 }
 
+void
+__dir_entry_fop_common_cbk (call_frame_t *frame, int child_index,
+                            xlator_t *this, int32_t op_ret,
+                            int32_t op_errno, inode_t *inode,
+                            struct iatt *buf, struct iatt *preparent,
+                            struct iatt *postparent, struct iatt *prenewparent,
+                            struct iatt *postnewparent)
+{
+        afr_local_t     *local          = NULL;
+
+        local = frame->local;
+
+        if (afr_fop_failed (op_ret, op_errno))
+                afr_transaction_fop_failed (frame, this, child_index);
+
+        if (op_ret > -1) {
+                local->op_ret = op_ret;
+
+                if ((local->success_count == 0) ||
+                    (child_index == local->read_child_index)) {
+                        local->cont.dir_fop.preparent      = *preparent;
+                        local->cont.dir_fop.postparent     = *postparent;
+                        if (buf)
+                                local->cont.dir_fop.buf            = *buf;
+                        if (prenewparent)
+                             local->cont.dir_fop.prenewparent  = *prenewparent;
+                        if (postnewparent)
+                             local->cont.dir_fop.postnewparent = *postnewparent;
+                }
+
+                local->cont.dir_fop.inode = inode;
+
+                local->fresh_children[local->success_count] = child_index;
+                local->success_count++;
+                local->child_errno[child_index] = 0;
+        } else {
+                local->child_errno[child_index] = op_errno;
+        }
+
+        local->op_errno = op_errno;
+}
+
+int
+afr_mark_new_entry_changelog_cbk (call_frame_t *frame, void *cookie,
+                                  xlator_t *this,
+                                  int32_t op_ret, int32_t op_errno,
+                                  dict_t *xattr, dict_t *xdata)
+{
+        int     call_count = 0;
+
+        call_count = afr_frame_return (frame);
+        if (call_count == 0) {
+                AFR_STACK_DESTROY (frame);
+        }
+        return 0;
+}
+
+void
+afr_mark_new_entry_changelog (call_frame_t *frame, xlator_t *this)
+{
+        call_frame_t  *new_frame  = NULL;
+        afr_local_t   *local      = NULL;
+        afr_local_t   *new_local  = NULL;
+        afr_private_t *priv       = NULL;
+        dict_t        **xattr     = NULL;
+        int32_t       **changelog = NULL;
+        int           i           = 0;
+        GF_UNUSED int op_errno    = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        new_frame = copy_frame (frame);
+        if (!new_frame) {
+                goto out;
+        }
+
+        AFR_LOCAL_ALLOC_OR_GOTO (new_frame->local, out);
+        new_local = new_frame->local;
+        changelog = afr_matrix_create (priv->child_count, AFR_NUM_CHANGE_LOGS);
+        if (!changelog)
+                goto out;
+
+        xattr = GF_CALLOC (priv->child_count, sizeof (*xattr),
+                           gf_afr_mt_dict_t);
+        if (!xattr)
+                goto out;
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_errno[i])
+                        continue;
+                xattr[i] = dict_new ();
+                if (!xattr[i])
+                        goto out;
+        }
+
+        afr_prepare_new_entry_pending_matrix (changelog,
+                                              afr_is_errno_set,
+                                              local->child_errno,
+                                              &local->cont.dir_fop.buf,
+                                              priv->child_count);
+
+        new_local->pending = changelog;
+        uuid_copy (new_local->loc.gfid, local->cont.dir_fop.buf.ia_gfid);
+        new_local->loc.inode = inode_ref (local->cont.dir_fop.inode);
+        new_local->call_count = local->success_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_errno[i])
+                        continue;
+
+                afr_set_pending_dict (priv, xattr[i], changelog, i, LOCAL_LAST);
+                STACK_WIND_COOKIE (new_frame, afr_mark_new_entry_changelog_cbk,
+                                   (void *) (long) i, priv->children[i],
+                                   priv->children[i]->fops->xattrop,
+                                   &new_local->loc, GF_XATTROP_ADD_ARRAY,
+                                   xattr[i], NULL);
+        }
+        new_frame = NULL;
+out:
+        if (new_frame)
+                AFR_STACK_DESTROY (new_frame);
+        afr_xattr_array_destroy (xattr, priv->child_count);
+        return;
+}
+
+gf_boolean_t
+afr_is_new_entry_changelog_needed (glusterfs_fop_t fop)
+{
+        glusterfs_fop_t fops[]   = {GF_FOP_CREATE, GF_FOP_MKNOD, GF_FOP_NULL};
+        int             i        = 0;
+
+        for (i = 0; fops[i] != GF_FOP_NULL; i++) {
+                if (fop == fops[i])
+                        return _gf_true;
+        }
+        return _gf_false;
+}
+
+void
+afr_dir_fop_mark_entry_pending_changelog (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t   *local      = NULL;
+        afr_private_t *priv       = NULL;
+
+        local = frame->local;
+        priv  = this->private;
+
+        if (local->op_ret < 0)
+                goto out;
+
+        if (local->success_count == priv->child_count)
+                goto out;
+
+        if (!afr_is_new_entry_changelog_needed (local->op))
+                goto out;
+
+        afr_mark_new_entry_changelog (frame, this);
+
+out:
+        return;
+}
+
+void
+afr_dir_fop_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local          = NULL;
+        afr_private_t   *priv           = NULL;
+
+        local = frame->local;
+        priv  = this->private;
+
+        if (local->cont.dir_fop.inode == NULL)
+                goto done;
+        afr_set_read_ctx_from_policy (this, local->cont.dir_fop.inode,
+                                      local->fresh_children,
+                                      local->read_child_index,
+                                      priv->read_child,
+                                      local->cont.dir_fop.buf.ia_gfid);
+done:
+        local->transaction.unwind (frame, this);
+        afr_dir_fop_mark_entry_pending_changelog (frame, this);
+        local->transaction.resume (frame, this);
+}
+
 /* {{{ create */
 
 int
@@ -81,7 +263,6 @@ afr_create_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt  *unwind_buf = NULL;
 
         local = frame->local;
 
@@ -95,18 +276,14 @@ afr_create_unwind (call_frame_t *frame, xlator_t *this)
         UNLOCK (&frame->lock);
 
         if (main_frame) {
-                if (local->cont.create.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.create.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.create.buf;
-                }
-
                 AFR_STACK_UNWIND (create, main_frame,
                                   local->op_ret, local->op_errno,
                                   local->cont.create.fd,
-                                  local->cont.create.inode,
-                                  unwind_buf, &local->cont.create.preparent,
-                                  &local->cont.create.postparent);
+                                  local->cont.dir_fop.inode,
+                                  &local->cont.dir_fop.buf,
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  local->xdata_rsp);
         }
 
         return 0;
@@ -117,32 +294,24 @@ int
 afr_create_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno,
                      fd_t *fd, inode_t *inode, struct iatt *buf,
-                     struct iatt *preparent, struct iatt *postparent)
+                     struct iatt *preparent, struct iatt *postparent,
+                     dict_t *xdata)
 {
         afr_local_t     *local = NULL;
-        afr_private_t   *priv  = NULL;
         uint64_t        ctx = 0;
         afr_fd_ctx_t    *fd_ctx = NULL;
         int             ret = 0;
         int             call_count = -1;
         int             child_index = -1;
-        int32_t         *fresh_children = NULL;
 
         local = frame->local;
-        priv  = this->private;
 
         child_index = (long) cookie;
 
         LOCK (&frame->lock);
         {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret = op_ret;
-
+                if (op_ret > -1) {
                         ret = afr_fd_ctx_set (this, fd);
-
                         if (ret < 0) {
                                 gf_log (this->name, GF_LOG_ERROR,
                                         "could not set ctx on fd=%p", fd);
@@ -153,7 +322,6 @@ afr_create_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         }
 
                         ret = fd_ctx_get (fd, this, &ctx);
-
                         if (ret < 0) {
                                 gf_log (this->name, GF_LOG_ERROR,
                                         "could not get fd ctx for fd=%p", fd);
@@ -167,23 +335,14 @@ afr_create_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
                         fd_ctx->flags                  = local->cont.create.flags;
 
-                        if (local->success_count == 0)
-                                local->cont.create.buf        = *buf;
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.create.read_child_buf = *buf;
-                                local->cont.create.preparent      = *preparent;
-                                local->cont.create.postparent     = *postparent;
-                        }
-
-                        local->cont.create.inode = inode;
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
+                        if (local->success_count == 0) {
+				if (xdata)
+					local->xdata_rsp = dict_ref(xdata);
+			}
                 }
-
-                local->op_errno = op_errno;
+                __dir_entry_fop_common_cbk (frame, child_index, this,
+                                            op_ret, op_errno, inode, buf,
+                                            preparent, postparent, NULL, NULL);
         }
 
 unlock:
@@ -191,15 +350,8 @@ unlock:
 
         call_count = afr_frame_return (frame);
 
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
         return 0;
 }
@@ -235,8 +387,9 @@ afr_create_wind (call_frame_t *frame, xlator_t *this)
                                            &local->loc,
                                            local->cont.create.flags,
                                            local->cont.create.mode,
+                                           local->umask,
                                            local->cont.create.fd,
-                                           local->cont.create.params);
+                                           local->xdata_req);
                         if (!--call_count)
                                 break;
                 }
@@ -264,13 +417,14 @@ afr_create_done (call_frame_t *frame, xlator_t *this)
 int
 afr_create (call_frame_t *frame, xlator_t *this,
             loc_t *loc, int32_t flags, mode_t mode,
-            fd_t *fd, dict_t *params)
+            mode_t umask, fd_t *fd, dict_t *params)
 {
-        afr_private_t  *priv  = NULL;
-        afr_local_t    *local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-        int             ret = -1;
-        int             op_errno = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -302,11 +456,13 @@ afr_create (call_frame_t *frame, xlator_t *this,
         }
         UNLOCK (&priv->read_child_lock);
 
+        local->op                = GF_FOP_CREATE;
         local->cont.create.flags = flags;
         local->cont.create.mode  = mode;
         local->cont.create.fd    = fd_ref (fd);
+        local->umask  = umask;
         if (params)
-                local->cont.create.params = dict_ref (params);
+                local->xdata_req = dict_ref (params);
 
         local->transaction.fop    = afr_create_wind;
         local->transaction.done   = afr_create_done;
@@ -319,8 +475,22 @@ afr_create (call_frame_t *frame, xlator_t *this,
 
         local->transaction.main_frame = frame;
         local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
@@ -328,7 +498,7 @@ out:
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
                 AFR_STACK_UNWIND (create, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
+                                  NULL, NULL, NULL, NULL, NULL, NULL);
         }
 
         return 0;
@@ -343,7 +513,6 @@ afr_mknod_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
 
         local = frame->local;
 
@@ -357,17 +526,13 @@ afr_mknod_unwind (call_frame_t *frame, xlator_t *this)
         UNLOCK (&frame->lock);
 
         if (main_frame) {
-                if (local->cont.mknod.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.mknod.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.mknod.buf;
-                }
-
                 AFR_STACK_UNWIND (mknod, main_frame,
                                   local->op_ret, local->op_errno,
-                                  local->cont.mknod.inode,
-                                  unwind_buf, &local->cont.mknod.preparent,
-                                  &local->cont.mknod.postparent);
+                                  local->cont.dir_fop.inode,
+                                  &local->cont.dir_fop.buf,
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  NULL);
         }
 
         return 0;
@@ -378,58 +543,25 @@ int
 afr_mknod_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
         int             call_count      = -1;
         int             child_index     = -1;
-        int32_t         *fresh_children = NULL;
-
-        local = frame->local;
-        priv = this->private;
 
         child_index = (long) cookie;
 
         LOCK (&frame->lock);
         {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret = op_ret;
-
-                        if (local->success_count == 0)
-                                local->cont.mknod.buf   = *buf;
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.mknod.read_child_buf = *buf;
-                                local->cont.mknod.preparent      = *preparent;
-                                local->cont.mknod.postparent     = *postparent;
-                        }
-
-                        local->cont.mknod.inode = inode;
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
+                __dir_entry_fop_common_cbk (frame, child_index, this,
+                                            op_ret, op_errno, inode, buf,
+                                            preparent, postparent, NULL, NULL);
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
         return 0;
 }
@@ -463,7 +595,8 @@ afr_mknod_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i]->fops->mknod,
                                            &local->loc, local->cont.mknod.mode,
                                            local->cont.mknod.dev,
-                                           local->cont.mknod.params);
+                                           local->umask,
+                                           local->xdata_req);
                         if (!--call_count)
                                 break;
                 }
@@ -488,14 +621,15 @@ afr_mknod_done (call_frame_t *frame, xlator_t *this)
 
 
 int
-afr_mknod (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, dev_t dev, dict_t *params)
+afr_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+           dev_t dev, mode_t umask, dict_t *params)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -527,10 +661,12 @@ afr_mknod (call_frame_t *frame, xlator_t *this,
         }
         UNLOCK (&priv->read_child_lock);
 
+        local->op               = GF_FOP_MKNOD;
         local->cont.mknod.mode  = mode;
         local->cont.mknod.dev   = dev;
+        local->umask = umask;
         if (params)
-                local->cont.mknod.params = dict_ref (params);
+                local->xdata_req = dict_ref (params);
 
         local->transaction.fop    = afr_mknod_wind;
         local->transaction.done   = afr_mknod_done;
@@ -543,8 +679,22 @@ afr_mknod (call_frame_t *frame, xlator_t *this,
 
         local->transaction.main_frame = frame;
         local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
@@ -552,7 +702,7 @@ out:
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
                 AFR_STACK_UNWIND (mknod, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL);
+                                  NULL, NULL, NULL, NULL, NULL);
         }
 
         return 0;
@@ -568,7 +718,6 @@ afr_mkdir_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
 
         local = frame->local;
 
@@ -582,17 +731,13 @@ afr_mkdir_unwind (call_frame_t *frame, xlator_t *this)
         UNLOCK (&frame->lock);
 
         if (main_frame) {
-                if (local->cont.mkdir.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.mkdir.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.mkdir.buf;
-                }
-
                 AFR_STACK_UNWIND (mkdir, main_frame,
                                   local->op_ret, local->op_errno,
-                                  local->cont.mkdir.inode,
-                                  unwind_buf, &local->cont.mkdir.preparent,
-                                  &local->cont.mkdir.postparent);
+                                  local->cont.dir_fop.inode,
+                                  &local->cont.dir_fop.buf,
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  NULL);
         }
 
         return 0;
@@ -603,58 +748,25 @@ int
 afr_mkdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
         int             call_count      = -1;
         int             child_index     = -1;
-        int32_t         *fresh_children = NULL;
-
-        local = frame->local;
-        priv = this->private;
 
         child_index = (long) cookie;
 
         LOCK (&frame->lock);
         {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret           = op_ret;
-
-                        if (local->success_count == 0)
-                                local->cont.mkdir.buf   = *buf;
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.mkdir.read_child_buf = *buf;
-                                local->cont.mkdir.preparent      = *preparent;
-                                local->cont.mkdir.postparent     = *postparent;
-                        }
-
-                        local->cont.mkdir.inode = inode;
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
-
-                local->op_errno         = op_errno;
+                __dir_entry_fop_common_cbk (frame, child_index, this,
+                                            op_ret, op_errno, inode, buf,
+                                            preparent, postparent, NULL, NULL);
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
         return 0;
 }
@@ -688,7 +800,8 @@ afr_mkdir_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i],
                                            priv->children[i]->fops->mkdir,
                                            &local->loc, local->cont.mkdir.mode,
-                                           local->cont.mkdir.params);
+                                           local->umask,
+                                           local->xdata_req);
                         if (!--call_count)
                                 break;
                 }
@@ -712,16 +825,16 @@ afr_mkdir_done (call_frame_t *frame, xlator_t *this)
         return 0;
 }
 
-
 int
 afr_mkdir (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, dict_t *params)
+           loc_t *loc, mode_t mode, mode_t umask, dict_t *params)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -754,9 +867,11 @@ afr_mkdir (call_frame_t *frame, xlator_t *this,
         UNLOCK (&priv->read_child_lock);
 
         local->cont.mkdir.mode  = mode;
+        local->umask = umask;
         if (params)
-                local->cont.mkdir.params = dict_ref (params);
+                local->xdata_req = dict_ref (params);
 
+        local->op = GF_FOP_MKDIR;
         local->transaction.fop    = afr_mkdir_wind;
         local->transaction.done   = afr_mkdir_done;
         local->transaction.unwind = afr_mkdir_unwind;
@@ -768,8 +883,22 @@ afr_mkdir (call_frame_t *frame, xlator_t *this,
 
         local->transaction.main_frame = frame;
         local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
@@ -778,7 +907,7 @@ out:
                         AFR_STACK_DESTROY (transaction_frame);
 
                 AFR_STACK_UNWIND (mkdir, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL);
+                                  NULL, NULL, NULL, NULL, NULL);
         }
 
         return 0;
@@ -794,7 +923,6 @@ afr_link_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
 
         local = frame->local;
 
@@ -808,17 +936,13 @@ afr_link_unwind (call_frame_t *frame, xlator_t *this)
         UNLOCK (&frame->lock);
 
         if (main_frame) {
-                if (local->cont.link.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.link.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.link.buf;
-                }
-
                 AFR_STACK_UNWIND (link, main_frame,
                                   local->op_ret, local->op_errno,
-                                  local->cont.link.inode,
-                                  unwind_buf, &local->cont.link.preparent,
-                                  &local->cont.link.postparent);
+                                  local->cont.dir_fop.inode,
+                                  &local->cont.dir_fop.buf,
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  NULL);
         }
 
         return 0;
@@ -829,59 +953,25 @@ int
 afr_link_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
         int             call_count      = -1;
         int             child_index     = -1;
-        int32_t         *fresh_children = NULL;
-
-        local = frame->local;
-        priv = this->private;
 
         child_index = (long) cookie;
 
         LOCK (&frame->lock);
         {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret   = op_ret;
-
-                        if (local->success_count == 0) {
-                                local->cont.link.buf        = *buf;
-                        }
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.link.read_child_buf = *buf;
-                                local->cont.link.preparent      = *preparent;
-                                local->cont.link.postparent     = *postparent;
-                        }
-
-                        local->cont.link.inode    = inode;
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
+                __dir_entry_fop_common_cbk (frame, child_index, this,
+                                            op_ret, op_errno, inode, buf,
+                                            preparent, postparent, NULL, NULL);
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
         return 0;
 }
@@ -910,11 +1000,12 @@ afr_link_wind (call_frame_t *frame, xlator_t *this)
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_link_wind_cbk, (void *) (long) i,
+                        STACK_WIND_COOKIE (frame, afr_link_wind_cbk,
+                                           (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->link,
                                            &local->loc,
-                                           &local->newloc);
+                                           &local->newloc, local->xdata_req);
 
                         if (!--call_count)
                                 break;
@@ -940,13 +1031,14 @@ afr_link_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_link (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc)
+          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -971,6 +1063,8 @@ afr_link (call_frame_t *frame, xlator_t *this,
 
         loc_copy (&local->loc,    oldloc);
         loc_copy (&local->newloc, newloc);
+        if (xdata)
+                local->xdata_req = dict_ref (xdata);
 
         LOCK (&priv->read_child_lock);
         {
@@ -979,6 +1073,7 @@ afr_link (call_frame_t *frame, xlator_t *this,
         }
         UNLOCK (&priv->read_child_lock);
 
+        local->op = GF_FOP_LINK;
         local->transaction.fop    = afr_link_wind;
         local->transaction.done   = afr_link_done;
         local->transaction.unwind = afr_link_unwind;
@@ -990,16 +1085,29 @@ afr_link (call_frame_t *frame, xlator_t *this,
 
         local->transaction.main_frame   = frame;
         local->transaction.basename     = AFR_BASENAME (newloc->path);
+        int_lock = &local->internal_lock;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
         ret = 0;
 out:
         if (ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
                 AFR_STACK_UNWIND (link, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL);
+                                  NULL, NULL, NULL, NULL, NULL);
         }
 
         return 0;
@@ -1015,7 +1123,6 @@ afr_symlink_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
 
         local = frame->local;
 
@@ -1029,17 +1136,13 @@ afr_symlink_unwind (call_frame_t *frame, xlator_t *this)
         UNLOCK (&frame->lock);
 
         if (main_frame) {
-                if (local->cont.symlink.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.symlink.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.symlink.buf;
-                }
-
                 AFR_STACK_UNWIND (symlink, main_frame,
                                   local->op_ret, local->op_errno,
-                                  local->cont.symlink.inode,
-                                  unwind_buf, &local->cont.symlink.preparent,
-                                  &local->cont.symlink.postparent);
+                                  local->cont.dir_fop.inode,
+                                  &local->cont.dir_fop.buf,
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  NULL);
         }
 
         return 0;
@@ -1050,58 +1153,25 @@ int
 afr_symlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, inode_t *inode,
                       struct iatt *buf, struct iatt *preparent,
-                      struct iatt *postparent)
+                      struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
         int             call_count      = -1;
         int             child_index     = -1;
-        int32_t         *fresh_children = NULL;
-
-        local = frame->local;
-        priv = this->private;
 
         child_index = (long) cookie;
 
         LOCK (&frame->lock);
         {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret   = op_ret;
-
-                        if (local->success_count == 0)
-                                local->cont.symlink.buf        = *buf;
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.symlink.read_child_buf = *buf;
-                                local->cont.symlink.preparent      = *preparent;
-                                local->cont.symlink.postparent     = *postparent;
-                        }
-
-                        local->cont.symlink.inode    = inode;
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
+                __dir_entry_fop_common_cbk (frame, child_index, this,
+                                            op_ret, op_errno, inode, buf,
+                                            preparent, postparent, NULL, NULL);
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
         return 0;
 }
@@ -1136,7 +1206,8 @@ afr_symlink_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i]->fops->symlink,
                                            local->cont.symlink.linkpath,
                                            &local->loc,
-                                           local->cont.symlink.params);
+                                           local->umask,
+                                           local->xdata_req);
 
                         if (!--call_count)
                                 break;
@@ -1163,13 +1234,14 @@ afr_symlink_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_symlink (call_frame_t *frame, xlator_t *this,
-             const char *linkpath, loc_t *loc, dict_t *params)
+             const char *linkpath, loc_t *loc, mode_t umask, dict_t *params)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -1202,9 +1274,11 @@ afr_symlink (call_frame_t *frame, xlator_t *this,
         UNLOCK (&priv->read_child_lock);
 
         local->cont.symlink.linkpath = gf_strdup (linkpath);
+        local->umask = umask;
         if (params)
-                local->cont.symlink.params = dict_ref (params);
+                local->xdata_req = dict_ref (params);
 
+        local->op = GF_FOP_SYMLINK;
         local->transaction.fop    = afr_symlink_wind;
         local->transaction.done   = afr_symlink_done;
         local->transaction.unwind = afr_symlink_unwind;
@@ -1216,8 +1290,22 @@ afr_symlink (call_frame_t *frame, xlator_t *this,
 
         local->transaction.main_frame   = frame;
         local->transaction.basename     = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
@@ -1225,7 +1313,7 @@ out:
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
                 AFR_STACK_UNWIND (symlink, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL);
+                                  NULL, NULL, NULL, NULL, NULL);
         }
 
         return 0;
@@ -1240,7 +1328,6 @@ afr_rename_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
 
         local = frame->local;
 
@@ -1254,19 +1341,14 @@ afr_rename_unwind (call_frame_t *frame, xlator_t *this)
         UNLOCK (&frame->lock);
 
         if (main_frame) {
-                if (local->cont.rename.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.rename.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.rename.buf;
-                }
-
                 AFR_STACK_UNWIND (rename, main_frame,
                                   local->op_ret, local->op_errno,
-                                  unwind_buf,
-                                  &local->cont.rename.preoldparent,
-                                  &local->cont.rename.postoldparent,
-                                  &local->cont.rename.prenewparent,
-                                  &local->cont.rename.postnewparent);
+                                  &local->cont.dir_fop.buf,
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  &local->cont.dir_fop.prenewparent,
+                                  &local->cont.dir_fop.postnewparent,
+                                  NULL);
         }
 
         return 0;
@@ -1277,7 +1359,8 @@ int
 afr_rename_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *buf,
                      struct iatt *preoldparent, struct iatt *postoldparent,
-                     struct iatt *prenewparent, struct iatt *postnewparent)
+                     struct iatt *prenewparent, struct iatt *postnewparent,
+                     dict_t *xdata)
 {
         afr_local_t *   local = NULL;
         int call_count = -1;
@@ -1291,38 +1374,22 @@ afr_rename_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         {
                 if (afr_fop_failed (op_ret, op_errno) && op_errno != ENOTEMPTY)
                         afr_transaction_fop_failed (frame, this, child_index);
+                local->op_errno = op_errno;
+                local->child_errno[child_index] = op_errno;
 
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-
-                                if (buf) {
-                                        local->cont.rename.buf = *buf;
-                                }
-
-                                local->success_count++;
-                        }
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.rename.read_child_buf = *buf;
-
-                                local->cont.rename.preoldparent  = *preoldparent;
-                                local->cont.rename.postoldparent = *postoldparent;
-                                local->cont.rename.prenewparent  = *prenewparent;
-                                local->cont.rename.postnewparent = *postnewparent;
-                        }
-                }
+                if (op_ret > -1)
+                        __dir_entry_fop_common_cbk (frame, child_index, this,
+                                                   op_ret, op_errno, NULL, buf,
+                                                   preoldparent, postoldparent,
+                                                   prenewparent, postnewparent);
 
-                local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
-        if (call_count == 0) {
-                local->transaction.unwind (frame, this);
-                local->transaction.resume (frame, this);
-        }
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
         return 0;
 }
@@ -1356,7 +1423,7 @@ afr_rename_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i],
                                            priv->children[i]->fops->rename,
                                            &local->loc,
-                                           &local->newloc);
+                                           &local->newloc, NULL);
                         if (!--call_count)
                                 break;
                 }
@@ -1381,13 +1448,15 @@ afr_rename_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_rename (call_frame_t *frame, xlator_t *this,
-            loc_t *oldloc, loc_t *newloc)
+            loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
+        int                     nlockee                 = 0;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -1415,6 +1484,7 @@ afr_rename (call_frame_t *frame, xlator_t *this,
 
         local->read_child_index = afr_inode_get_read_ctx (this, oldloc->inode, NULL);
 
+        local->op = GF_FOP_RENAME;
         local->transaction.fop    = afr_rename_wind;
         local->transaction.done   = afr_rename_done;
         local->transaction.unwind = afr_rename_unwind;
@@ -1431,8 +1501,44 @@ afr_rename (call_frame_t *frame, xlator_t *this,
         local->transaction.main_frame   = frame;
         local->transaction.basename     = AFR_BASENAME (oldloc->path);
         local->transaction.new_basename = AFR_BASENAME (newloc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = nlockee = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->transaction.new_parent_loc,
+                                     local->transaction.new_basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        nlockee++;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_RENAME_TRANSACTION);
+        nlockee++;
+        if (local->newloc.inode && IA_ISDIR (local->newloc.inode->ia_type)) {
+                ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                             &local->newloc,
+                                             NULL,
+                                             priv->child_count);
+                if (ret)
+                        goto out;
+
+                nlockee++;
+        }
+        qsort (int_lock->lockee, nlockee, sizeof (*int_lock->lockee),
+               afr_entry_lockee_cmp);
+        int_lock->lockee_count = nlockee;
+
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_RENAME_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
@@ -1441,7 +1547,7 @@ out:
                         AFR_STACK_DESTROY (transaction_frame);
 
                 AFR_STACK_UNWIND (rename, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
+                                  NULL, NULL, NULL, NULL, NULL, NULL);
         }
 
         return 0;
@@ -1471,8 +1577,9 @@ afr_unlink_unwind (call_frame_t *frame, xlator_t *this)
         if (main_frame) {
                 AFR_STACK_UNWIND (unlink, main_frame,
                                   local->op_ret, local->op_errno,
-                                  &local->cont.unlink.preparent,
-                                  &local->cont.unlink.postparent);
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  NULL);
         }
 
         return 0;
@@ -1482,7 +1589,7 @@ afr_unlink_unwind (call_frame_t *frame, xlator_t *this)
 int
 afr_unlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                     struct iatt *postparent)
+                     struct iatt *postparent, dict_t *xdata)
 {
         afr_local_t *   local = NULL;
         int call_count  = -1;
@@ -1495,36 +1602,15 @@ afr_unlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (child_index == local->read_child_index) {
                         local->read_child_returned = _gf_true;
                 }
-
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret   = op_ret;
-                                local->cont.unlink.preparent  = *preparent;
-                                local->cont.unlink.postparent = *postparent;
-                        }
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.unlink.preparent  = *preparent;
-                                local->cont.unlink.postparent = *postparent;
-                        }
-
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
+                __dir_entry_fop_common_cbk (frame, child_index, this,
+                                            op_ret, op_errno, NULL, NULL,
+                                            preparent, postparent, NULL, NULL);
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
         return 0;
 }
@@ -1557,7 +1643,8 @@ afr_unlink_wind (call_frame_t *frame, xlator_t *this)
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->unlink,
-                                           &local->loc);
+                                           &local->loc, local->xflag,
+                                           local->xdata_req);
 
                         if (!--call_count)
                                 break;
@@ -1583,13 +1670,14 @@ afr_unlink_done (call_frame_t *frame, xlator_t *this)
 
 int32_t
 afr_unlink (call_frame_t *frame, xlator_t *this,
-            loc_t *loc)
+            loc_t *loc, int xflag, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -1613,7 +1701,11 @@ afr_unlink (call_frame_t *frame, xlator_t *this,
                 goto out;
 
         loc_copy (&local->loc, loc);
+        local->xflag = xflag;
+        if (xdata)
+                local->xdata_req = dict_ref (xdata);
 
+        local->op = GF_FOP_UNLINK;
         local->transaction.fop    = afr_unlink_wind;
         local->transaction.done   = afr_unlink_done;
         local->transaction.unwind = afr_unlink_unwind;
@@ -1625,8 +1717,22 @@ afr_unlink (call_frame_t *frame, xlator_t *this,
 
         local->transaction.main_frame = frame;
         local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
@@ -1634,7 +1740,7 @@ out:
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
                 AFR_STACK_UNWIND (unlink, frame, -1, op_errno,
-                                  NULL, NULL);
+                                  NULL, NULL, NULL);
         }
 
         return 0;
@@ -1666,8 +1772,9 @@ afr_rmdir_unwind (call_frame_t *frame, xlator_t *this)
         if (main_frame) {
                 AFR_STACK_UNWIND (rmdir, main_frame,
                                   local->op_ret, local->op_errno,
-                                  &local->cont.rmdir.preparent,
-                                  &local->cont.rmdir.postparent);
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  NULL);
         }
 
         return 0;
@@ -1677,7 +1784,7 @@ afr_rmdir_unwind (call_frame_t *frame, xlator_t *this)
 int
 afr_rmdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
         afr_local_t *   local = NULL;
         int call_count  = -1;
@@ -1691,36 +1798,22 @@ afr_rmdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (child_index == read_child) {
                         local->read_child_returned = _gf_true;
                 }
-
                 if (afr_fop_failed (op_ret, op_errno) && (op_errno != ENOTEMPTY))
                         afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.rmdir.preparent  = *preparent;
-                                local->cont.rmdir.postparent = *postparent;
-
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.rmdir.preparent  = *preparent;
-                                local->cont.rmdir.postparent = *postparent;
-                        }
-
-                        local->success_count++;
-                }
-
                 local->op_errno = op_errno;
+                local->child_errno[child_index] = op_errno;
+                if (op_ret > -1)
+                        __dir_entry_fop_common_cbk (frame, child_index, this,
+                                                   op_ret, op_errno, NULL, NULL,
+                                                   preparent, postparent, NULL,
+                                                   NULL);
+
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.unwind (frame, this);
-                local->transaction.resume (frame, this);
-        }
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
         return 0;
 }
@@ -1753,7 +1846,8 @@ afr_rmdir_wind (call_frame_t *frame, xlator_t *this)
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->rmdir,
-                                           &local->loc, local->cont.rmdir.flags);
+                                           &local->loc, local->cont.rmdir.flags,
+                                           NULL);
 
                         if (!--call_count)
                                 break;
@@ -1779,13 +1873,15 @@ afr_rmdir_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_rmdir (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, int flags)
+           loc_t *loc, int flags, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
+        int                     nlockee                 = 0;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -1811,6 +1907,7 @@ afr_rmdir (call_frame_t *frame, xlator_t *this,
         local->cont.rmdir.flags = flags;
         loc_copy (&local->loc, loc);
 
+        local->op = GF_FOP_RMDIR;
         local->transaction.fop    = afr_rmdir_wind;
         local->transaction.done   = afr_rmdir_done;
         local->transaction.unwind = afr_rmdir_unwind;
@@ -1822,15 +1919,41 @@ afr_rmdir (call_frame_t *frame, xlator_t *this,
 
         local->transaction.main_frame = frame;
         local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count = nlockee = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        nlockee++;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->loc,
+                                     NULL,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        nlockee++;
+        qsort (int_lock->lockee, nlockee, sizeof (*int_lock->lockee),
+               afr_entry_lockee_cmp);
+        int_lock->lockee_count = nlockee;
+
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
         if (ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL);
+                AFR_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
         }
 
         return 0;
diff --git a/xlators/cluster/afr/src/afr-dir-write.h b/xlators/cluster/afr/src/afr-dir-write.h
index 0290c6350..02f0a3682 100644
--- a/xlators/cluster/afr/src/afr-dir-write.h
+++ b/xlators/cluster/afr/src/afr-dir-write.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __DIR_WRITE_H__
@@ -23,38 +14,34 @@
 int32_t
 afr_create (call_frame_t *frame, xlator_t *this,
             loc_t *loc, int32_t flags, mode_t mode,
-            fd_t *fd, dict_t *params);
+            mode_t umask, fd_t *fd, dict_t *xdata);
 
 int32_t
 afr_mknod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dev_t dev, dict_t *params);
+	   loc_t *loc, mode_t mode, dev_t dev, mode_t umask, dict_t *xdata);
 
 int32_t
 afr_mkdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dict_t *params);
+	   loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata);
 
 int32_t
 afr_unlink (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc);
+	    loc_t *loc, int xflag, dict_t *xdata);
 
 int32_t
 afr_rmdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, int flags);
+	   loc_t *loc, int flags, dict_t *xdata);
 
 int32_t
 afr_link (call_frame_t *frame, xlator_t *this,
-	  loc_t *oldloc, loc_t *newloc);
+	  loc_t *oldloc, loc_t *newloc, dict_t *xdata);
 
 int32_t
 afr_rename (call_frame_t *frame, xlator_t *this,
-	    loc_t *oldloc, loc_t *newloc);
+	    loc_t *oldloc, loc_t *newloc, dict_t *xdata);
 
 int
 afr_symlink (call_frame_t *frame, xlator_t *this,
-	     const char *linkpath, loc_t *oldloc, dict_t *params);
-
-int32_t
-afr_setdents (call_frame_t *frame, xlator_t *this,
-	      fd_t *fd, int32_t flags, dir_entry_t *entries, int32_t count);
+	     const char *linkpath, loc_t *oldloc, mode_t umask, dict_t *params);
 
 #endif /* __DIR_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-inode-read.c b/xlators/cluster/afr/src/afr-inode-read.c
index 6c837c337..e06e3b2f2 100644
--- a/xlators/cluster/afr/src/afr-inode-read.c
+++ b/xlators/cluster/afr/src/afr-inode-read.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -58,7 +49,7 @@
 
 int32_t
 afr_access_cbk (call_frame_t *frame, void *cookie,
-                xlator_t *this, int32_t op_ret, int32_t op_errno)
+                xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_private_t * priv            = NULL;
         afr_local_t *   local           = NULL;
@@ -92,12 +83,13 @@ afr_access_cbk (call_frame_t *frame, void *cookie,
                                    (void *) (long) read_child,
                                    children[next_call_child],
                                    children[next_call_child]->fops->access,
-                                   &local->loc, local->cont.access.mask);
+                                   &local->loc, local->cont.access.mask,
+                                   NULL);
         }
 
 out:
         if (unwind) {
-                AFR_STACK_UNWIND (access, frame, op_ret, op_errno);
+                AFR_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
         }
 
         return 0;
@@ -105,7 +97,8 @@ out:
 
 
 int32_t
-afr_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
+afr_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+            dict_t *xdata)
 {
         afr_private_t   *priv      = NULL;
         xlator_t        **children = NULL;
@@ -124,6 +117,8 @@ afr_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
 
         children = priv->children;
 
+        AFR_SBRAIN_CHECK_LOC (loc, out);
+
         AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
@@ -156,12 +151,12 @@ afr_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
                            (void *) (long) call_child,
                            children[call_child],
                            children[call_child]->fops->access,
-                           loc, mask);
+                           loc, mask, xdata);
 
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (access, frame, -1, op_errno);
+                AFR_STACK_UNWIND (access, frame, -1, op_errno, NULL);
         return 0;
 }
 
@@ -173,7 +168,7 @@ out:
 int32_t
 afr_stat_cbk (call_frame_t *frame, void *cookie,
               xlator_t *this, int32_t op_ret, int32_t op_errno,
-              struct iatt *buf)
+              struct iatt *buf, dict_t *xdata)
 {
         afr_private_t * priv            = NULL;
         afr_local_t *   local           = NULL;
@@ -207,12 +202,12 @@ afr_stat_cbk (call_frame_t *frame, void *cookie,
                                    (void *) (long) read_child,
                                    children[next_call_child],
                                    children[next_call_child]->fops->stat,
-                                   &local->loc);
+                                   &local->loc, NULL);
         }
 
 out:
         if (unwind) {
-                AFR_STACK_UNWIND (stat, frame, op_ret, op_errno, buf);
+                AFR_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
         }
 
         return 0;
@@ -220,7 +215,7 @@ out:
 
 
 int32_t
-afr_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+afr_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         afr_private_t   *priv      = NULL;
         afr_local_t     *local     = NULL;
@@ -239,6 +234,8 @@ afr_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
 
         children = priv->children;
 
+        AFR_SBRAIN_CHECK_LOC (loc, out);
+
         AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
@@ -267,12 +264,12 @@ afr_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
         STACK_WIND_COOKIE (frame, afr_stat_cbk, (void *) (long) call_child,
                            children[call_child],
                            children[call_child]->fops->stat,
-                           loc);
+                           loc, xdata);
 
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (stat, frame, -1, op_errno, NULL);
+                AFR_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -284,7 +281,8 @@ out:
 
 int32_t
 afr_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *buf)
+               int32_t op_ret, int32_t op_errno, struct iatt *buf,
+               dict_t *xdata)
 {
         afr_private_t   *priv           = NULL;
         afr_local_t     *local          = NULL;
@@ -318,12 +316,12 @@ afr_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                    (void *) (long) read_child,
                                    children[next_call_child],
                                    children[next_call_child]->fops->fstat,
-                                   local->fd);
+                                   local->fd, NULL);
         }
 
 out:
         if (unwind) {
-                AFR_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf);
+                AFR_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
         }
 
         return 0;
@@ -332,7 +330,7 @@ out:
 
 int32_t
 afr_fstat (call_frame_t *frame, xlator_t *this,
-           fd_t *fd)
+           fd_t *fd, dict_t *xdata)
 {
         afr_private_t   *priv      = NULL;
         afr_local_t     *local     = NULL;
@@ -354,6 +352,8 @@ afr_fstat (call_frame_t *frame, xlator_t *this,
 
         VALIDATE_OR_GOTO (fd->inode, out);
 
+        AFR_SBRAIN_CHECK_FD (fd, out);
+
         AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
@@ -383,20 +383,17 @@ afr_fstat (call_frame_t *frame, xlator_t *this,
 
         local->fd = fd_ref (fd);
 
-        ret = afr_open_fd_fix (frame, this, _gf_false);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
+        afr_open_fd_fix (fd, this);
+
         STACK_WIND_COOKIE (frame, afr_fstat_cbk, (void *) (long) call_child,
                            children[call_child],
                            children[call_child]->fops->fstat,
-                           fd);
+                           fd, xdata);
 
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (fstat, frame, -1, op_errno, NULL);
+                AFR_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -408,7 +405,7 @@ out:
 int32_t
 afr_readlink_cbk (call_frame_t *frame, void *cookie,
                   xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  const char *buf, struct iatt *sbuf)
+                  const char *buf, struct iatt *sbuf, dict_t *xdata)
 {
         afr_private_t * priv                  = NULL;
         afr_local_t *   local                 = NULL;
@@ -442,12 +439,13 @@ afr_readlink_cbk (call_frame_t *frame, void *cookie,
                                    children[next_call_child],
                                    children[next_call_child]->fops->readlink,
                                    &local->loc,
-                                   local->cont.readlink.size);
+                                   local->cont.readlink.size, NULL);
         }
 
 out:
         if (unwind) {
-                AFR_STACK_UNWIND (readlink, frame, op_ret, op_errno, buf, sbuf);
+                AFR_STACK_UNWIND (readlink, frame, op_ret, op_errno, buf, sbuf,
+                                  xdata);
         }
 
         return 0;
@@ -456,7 +454,7 @@ out:
 
 int32_t
 afr_readlink (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, size_t size)
+              loc_t *loc, size_t size, dict_t *xdata)
 {
         afr_private_t   *priv      = NULL;
         xlator_t        **children = NULL;
@@ -475,6 +473,8 @@ afr_readlink (call_frame_t *frame, xlator_t *this,
 
         children = priv->children;
 
+        AFR_SBRAIN_CHECK_LOC (loc, out);
+
         AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
@@ -506,12 +506,12 @@ afr_readlink (call_frame_t *frame, xlator_t *this,
                            (void *) (long) call_child,
                            children[call_child],
                            children[call_child]->fops->readlink,
-                           loc, size);
+                           loc, size, xdata);
 
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL);
+                AFR_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -526,7 +526,7 @@ struct _xattr_key {
 };
 
 
-void
+int
 __gather_xattr_keys (dict_t *dict, char *key, data_t *value,
                      void *data)
 {
@@ -538,13 +538,14 @@ __gather_xattr_keys (dict_t *dict, char *key, data_t *value,
 
                 xkey = GF_CALLOC (1, sizeof (*xkey), gf_afr_mt_xattr_key);
                 if (!xkey)
-                        return;
+                        return -1;
 
                 xkey->key = key;
                 INIT_LIST_HEAD (&xkey->list);
 
                 list_add_tail (&xkey->list, list);
         }
+        return 0;
 }
 
 
@@ -574,7 +575,7 @@ __filter_xattrs (dict_t *dict)
 int32_t
 afr_getxattr_cbk (call_frame_t *frame, void *cookie,
                   xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  dict_t *dict)
+                  dict_t *dict, dict_t *xdata)
 {
         afr_private_t * priv            = NULL;
         afr_local_t *   local           = NULL;
@@ -608,7 +609,8 @@ afr_getxattr_cbk (call_frame_t *frame, void *cookie,
                                    children[next_call_child],
                                    children[next_call_child]->fops->getxattr,
                                    &local->loc,
-                                   local->cont.getxattr.name);
+                                   local->cont.getxattr.name,
+                                   NULL);
         }
 
 out:
@@ -616,25 +618,25 @@ out:
                 if (op_ret >= 0 && dict)
                         __filter_xattrs (dict);
 
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
         }
 
         return 0;
 }
 
 int32_t
-afr_getxattr_unwind (call_frame_t *frame,
-                     int op_ret, int op_errno, dict_t *dict)
+afr_getxattr_unwind (call_frame_t *frame, int op_ret, int op_errno,
+                     dict_t *dict, dict_t *xdata)
 
 {
-        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 int32_t
-afr_getxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
-                        xlator_t *this, int32_t op_ret, int32_t op_errno,
-                        dict_t *dict)
+afr_fgetxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
+                         xlator_t *this, int32_t op_ret, int32_t op_errno,
+                         dict_t *dict, dict_t *xdata)
 {
         afr_local_t     *local                  = NULL;
         afr_private_t   *priv                   = NULL;
@@ -656,6 +658,8 @@ afr_getxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
         LOCK (&frame->lock);
         {
                 callcnt = --local->call_count;
+                if (op_ret == -1)
+                        local->child_errno[cky] = op_errno;
 
                 if (!local->dict)
                         local->dict = dict_new ();
@@ -664,14 +668,102 @@ afr_getxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
                                             &tmp_report);
                         if (ret)
                                 goto unlock;
-                        ret = dict_set_str (local->dict,
-                                            children[cky]->name,
-                                            tmp_report);
+                        ret = dict_set_dynstr (local->dict,
+                                               children[cky]->name,
+                                               gf_strdup (tmp_report));
                         if (ret)
                                 goto unlock;
                 }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                xattr = dict_new ();
+                if (!xattr) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        goto unwind;
+                }
+                ret = dict_serialize_value_with_delim (local->dict,
+                                                       lk_summary,
+                                                       &serz_len, '\n');
+                if (ret) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error serializing dictionary");
+                        goto unwind;
+                }
+                if (serz_len == -1)
+                        snprintf (lk_summary, sizeof (lk_summary),
+                                  "No locks cleared.");
+                ret = dict_set_dynstr (xattr, local->cont.getxattr.name,
+                                       gf_strdup (lk_summary));
+                if (ret) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error setting dictionary");
+                        goto unwind;
+                }
+
+        unwind:
+                // Updating child_errno with more recent 'events'
+                local->child_errno[cky] = op_errno;
+                op_errno = afr_resultant_errno_get (NULL, local->child_errno,
+                                                    priv->child_count);
+                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, xattr,
+                                  xdata);
+
+                if (xattr)
+                        dict_unref (xattr);
+        }
+
+        return ret;
+}
+
+int32_t
+afr_getxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
+                        xlator_t *this, int32_t op_ret, int32_t op_errno,
+                        dict_t *dict, dict_t *xdata)
+{
+        afr_local_t     *local                  = NULL;
+        afr_private_t   *priv                   = NULL;
+        xlator_t        **children              = NULL;
+        dict_t          *xattr                  = NULL;
+        char            *tmp_report             = NULL;
+        char            lk_summary[1024]        = {0,};
+        int             serz_len                = 0;
+        int32_t         callcnt                 = 0;
+        long int        cky                     = 0;
+        int             ret                     = 0;
+
+        priv     = this->private;
+        children = priv->children;
+
+        local = frame->local;
+        cky = (long) cookie;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
                 if (op_ret == -1)
                         local->child_errno[cky] = op_errno;
+
+                if (!local->dict)
+                        local->dict = dict_new ();
+                if (local->dict) {
+                        ret = dict_get_str (dict, local->cont.getxattr.name,
+                                            &tmp_report);
+                        if (ret)
+                                goto unlock;
+                        ret = dict_set_dynstr (local->dict,
+                                               children[cky]->name,
+                                               gf_strdup (tmp_report));
+                        if (ret)
+                                goto unlock;
+                }
         }
 unlock:
         UNLOCK (&frame->lock);
@@ -711,7 +803,7 @@ unlock:
                 local->child_errno[cky] = op_errno;
                 op_errno = afr_resultant_errno_get (NULL, local->child_errno,
                                                     priv->child_count);
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr);
+                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
 
                 if (xattr)
                         dict_unref (xattr);
@@ -726,7 +818,7 @@ unlock:
 int32_t
 afr_getxattr_node_uuid_cbk (call_frame_t *frame, void *cookie,
                             xlator_t *this, int32_t op_ret, int32_t op_errno,
-                            dict_t *dict)
+                            dict_t *dict, dict_t *xdata)
 {
         afr_private_t  *priv            = NULL;
         afr_local_t    *local           = NULL;
@@ -759,20 +851,373 @@ afr_getxattr_node_uuid_cbk (call_frame_t *frame, void *cookie,
                                    children[curr_call_child],
                                    children[curr_call_child]->fops->getxattr,
                                    &local->loc,
-                                   local->cont.getxattr.name);
+                                   local->cont.getxattr.name,
+                                   NULL);
         }
 
  unwind:
         if (unwind)
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict,
+                                  NULL);
 
         return 0;
 }
 
 int32_t
+afr_getxattr_lockinfo_cbk (call_frame_t *frame, void *cookie,
+                           xlator_t *this, int32_t op_ret, int32_t op_errno,
+                           dict_t *dict, dict_t *xdata)
+{
+        int          call_cnt     = 0, len = 0;
+        char        *lockinfo_buf = NULL;
+        dict_t      *lockinfo     = NULL, *newdict = NULL;
+        afr_local_t *local        = NULL;
+
+        LOCK (&frame->lock);
+        {
+                local = frame->local;
+
+                call_cnt = --local->call_count;
+
+                if ((op_ret < 0) || (!dict && !xdata)) {
+                        goto unlock;
+                }
+
+                if (xdata) {
+                        if (!local->xdata_rsp) {
+                                local->xdata_rsp = dict_new ();
+                                if (!local->xdata_rsp) {
+                                        local->op_ret = -1;
+                                        local->op_errno = ENOMEM;
+                                        goto unlock;
+                                }
+                        }
+                }
+
+                if (!dict) {
+                        goto unlock;
+                }
+
+                op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
+                                               (void **)&lockinfo_buf, &len);
+
+                if (!lockinfo_buf) {
+                        goto unlock;
+                }
+
+                if (!local->dict) {
+                        local->dict = dict_new ();
+                        if (!local->dict) {
+                                local->op_ret = -1;
+                                local->op_errno = ENOMEM;
+                                goto unlock;
+                        }
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (lockinfo_buf != NULL) {
+                lockinfo = dict_new ();
+                if (lockinfo == NULL) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                } else {
+                        op_ret = dict_unserialize (lockinfo_buf, len,
+                                                   &lockinfo);
+
+                        if (lockinfo && local->dict) {
+                                dict_copy (lockinfo, local->dict);
+                        }
+                }
+        }
+
+        if (xdata && local->xdata_rsp) {
+                dict_copy (xdata, local->xdata_rsp);
+        }
+
+        if (!call_cnt) {
+                newdict = dict_new ();
+                if (!newdict) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                len = dict_serialized_length (local->dict);
+                if (len == 0) {
+                        goto unwind;
+                }
+
+                lockinfo_buf = GF_CALLOC (1, len, gf_common_mt_char);
+                if (!lockinfo_buf) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                op_ret = dict_serialize (local->dict, lockinfo_buf);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                }
+
+                op_ret = dict_set_dynptr (newdict, GF_XATTR_LOCKINFO_KEY,
+                                          (void *)lockinfo_buf, len);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                        goto unwind;
+                }
+
+        unwind:
+                AFR_STACK_UNWIND (getxattr, frame, op_ret,
+                                  op_errno, newdict,
+                                  local->xdata_rsp);
+        }
+
+        dict_unref (lockinfo);
+
+        return 0;
+}
+
+int32_t
+afr_fgetxattr_lockinfo_cbk (call_frame_t *frame, void *cookie,
+                            xlator_t *this, int32_t op_ret, int32_t op_errno,
+                            dict_t *dict, dict_t *xdata)
+{
+        int          call_cnt     = 0, len = 0;
+        char        *lockinfo_buf = NULL;
+        dict_t      *lockinfo     = NULL, *newdict = NULL;
+        afr_local_t *local        = NULL;
+
+        LOCK (&frame->lock);
+        {
+                local = frame->local;
+
+                call_cnt = --local->call_count;
+
+                if ((op_ret < 0) || (!dict && !xdata)) {
+                        goto unlock;
+                }
+
+                if (xdata) {
+                        if (!local->xdata_rsp) {
+                                local->xdata_rsp = dict_new ();
+                                if (!local->xdata_rsp) {
+                                        local->op_ret = -1;
+                                        local->op_errno = ENOMEM;
+                                        goto unlock;
+                                }
+                        }
+                }
+
+                if (!dict) {
+                        goto unlock;
+                }
+
+                op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
+                                               (void **)&lockinfo_buf, &len);
+
+                if (!lockinfo_buf) {
+                        goto unlock;
+                }
+
+                if (!local->dict) {
+                        local->dict = dict_new ();
+                        if (!local->dict) {
+                                local->op_ret = -1;
+                                local->op_errno = ENOMEM;
+                                goto unlock;
+                        }
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (lockinfo_buf != NULL) {
+                lockinfo = dict_new ();
+                if (lockinfo == NULL) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                } else {
+                        op_ret = dict_unserialize (lockinfo_buf, len,
+                                                   &lockinfo);
+
+                        if (lockinfo && local->dict) {
+                                dict_copy (lockinfo, local->dict);
+                        }
+                }
+        }
+
+        if (xdata && local->xdata_rsp) {
+                dict_copy (xdata, local->xdata_rsp);
+        }
+
+        if (!call_cnt) {
+                newdict = dict_new ();
+                if (!newdict) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                len = dict_serialized_length (local->dict);
+                if (len <= 0) {
+                        goto unwind;
+                }
+
+                lockinfo_buf = GF_CALLOC (1, len, gf_common_mt_char);
+                if (!lockinfo_buf) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                op_ret = dict_serialize (local->dict, lockinfo_buf);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                }
+
+                op_ret = dict_set_dynptr (newdict, GF_XATTR_LOCKINFO_KEY,
+                                          (void *)lockinfo_buf, len);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                        goto unwind;
+                }
+
+        unwind:
+                AFR_STACK_UNWIND (fgetxattr, frame, op_ret,
+                                  op_errno, newdict,
+                                  local->xdata_rsp);
+        }
+
+        dict_unref (lockinfo);
+
+        return 0;
+}
+
+int32_t
+afr_fgetxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
+                            xlator_t *this, int32_t op_ret, int32_t op_errno,
+                            dict_t *dict, dict_t *xdata)
+{
+        afr_local_t *local          = NULL;
+        int32_t      callcnt        = 0;
+        int          ret            = 0;
+        char        *xattr          = NULL;
+        char        *xattr_serz     = NULL;
+        char        xattr_cky[1024] = {0,};
+        dict_t      *nxattr         = NULL;
+        long         cky            = 0;
+        int32_t      padding        = 0;
+        int32_t      tlen           = 0;
+
+        if (!frame || !frame->local || !this) {
+                gf_log ("", GF_LOG_ERROR, "possible NULL deref");
+                goto out;
+        }
+
+        local = frame->local;
+        cky = (long) cookie;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (!dict || (op_ret < 0))
+                        goto out;
+
+                if (!local->dict)
+                        local->dict = dict_new ();
+
+                if (local->dict) {
+                        ret = dict_get_str (dict,
+                                            local->cont.getxattr.name,
+                                            &xattr);
+                        if (ret)
+                                goto out;
+
+                        xattr = gf_strdup (xattr);
+
+                        (void)snprintf (xattr_cky, 1024, "%s-%ld",
+                                        local->cont.getxattr.name, cky);
+                        ret = dict_set_dynstr (local->dict,
+                                               xattr_cky, xattr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Cannot set xattr cookie key");
+                                goto out;
+                        }
+
+                        local->cont.getxattr.xattr_len
+                                += strlen (xattr) + 1;
+                }
+        }
+out:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (!local->cont.getxattr.xattr_len)
+                        goto unwind;
+
+                nxattr = dict_new ();
+                if (!nxattr)
+                        goto unwind;
+
+                /* extra bytes for decorations (brackets and <>'s) */
+                padding += strlen (this->name)
+                        + strlen (AFR_PATHINFO_HEADER) + 4;
+                local->cont.getxattr.xattr_len += (padding + 2);
+
+                xattr_serz = GF_CALLOC (local->cont.getxattr.xattr_len,
+                                        sizeof (char), gf_common_mt_char);
+
+                if (!xattr_serz)
+                        goto unwind;
+
+                /* the xlator info */
+                (void) sprintf (xattr_serz, "(<"AFR_PATHINFO_HEADER"%s> ",
+                                this->name);
+
+                /* actual series of pathinfo */
+                ret = dict_serialize_value_with_delim (local->dict,
+                                                       xattr_serz
+                                                       + strlen (xattr_serz),
+                                                       &tlen, ' ');
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Error serializing"
+                                " dictionary");
+                        goto unwind;
+                }
+
+                /* closing part */
+                *(xattr_serz + padding + tlen) = ')';
+                *(xattr_serz + padding + tlen + 1) = '\0';
+
+                ret = dict_set_dynstr (nxattr, local->cont.getxattr.name,
+                                       xattr_serz);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR, "Cannot set pathinfo"
+                                " key in dict");
+
+        unwind:
+                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, nxattr,
+                                  xdata);
+
+                if (nxattr)
+                        dict_unref (nxattr);
+        }
+
+        return ret;
+}
+
+int32_t
 afr_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
                            xlator_t *this, int32_t op_ret, int32_t op_errno,
-                           dict_t *dict)
+                           dict_t *dict, dict_t *xdata)
 {
         afr_local_t *local          = NULL;
         int32_t      callcnt        = 0;
@@ -871,7 +1316,8 @@ afr_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
                                 " key in dict");
 
         unwind:
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, nxattr);
+                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, nxattr,
+                                  xdata);
 
                 if (nxattr)
                         dict_unref (nxattr);
@@ -880,8 +1326,65 @@ afr_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
         return ret;
 }
 
+static int
+afr_aggregate_stime_xattr (dict_t *this, char *key, data_t *value, void *data)
+{
+        int ret = 0;
+
+        if (fnmatch (GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0)
+                ret = gf_get_min_stime (THIS, data, key, value);
+
+        return ret;
+}
+
+int32_t
+afr_common_getxattr_stime_cbk (call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               dict_t *dict, dict_t *xdata)
+{
+        afr_local_t *local          = NULL;
+        int32_t      callcnt        = 0;
+
+        if (!frame || !frame->local || !this) {
+                gf_log ("", GF_LOG_ERROR, "possible NULL deref");
+                goto out;
+        }
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (!dict || (op_ret < 0)) {
+                        local->op_errno = op_errno;
+                        goto cleanup;
+                }
+
+                if (!local->dict)
+                        local->dict = dict_copy_with_ref (dict, NULL);
+                else
+                        dict_foreach (dict, afr_aggregate_stime_xattr,
+                                      local->dict);
+                local->op_ret = 0;
+        }
+
+cleanup:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
+                                  local->op_errno, local->dict, xdata);
+        }
+
+out:
+        return 0;
+}
+
+
 static gf_boolean_t
-afr_is_special_xattr (const char *name, fop_getxattr_cbk_t *cbk)
+afr_is_special_xattr (const char *name, fop_getxattr_cbk_t *cbk,
+                      gf_boolean_t is_fgetxattr)
 {
         gf_boolean_t    is_spl = _gf_true;
 
@@ -891,14 +1394,31 @@ afr_is_special_xattr (const char *name, fop_getxattr_cbk_t *cbk)
                 goto out;
         }
 
-        if (!strcmp (name, GF_XATTR_PATHINFO_KEY))
-                *cbk = afr_getxattr_pathinfo_cbk;
-
-        else if (!strncmp (name, GF_XATTR_CLRLK_CMD,
-                           strlen (GF_XATTR_CLRLK_CMD)))
-                *cbk = afr_getxattr_clrlk_cbk;
-        else
+        if (!strcmp (name, GF_XATTR_PATHINFO_KEY)) {
+                if (is_fgetxattr) {
+                        *cbk = afr_fgetxattr_pathinfo_cbk;
+                } else {
+                        *cbk = afr_getxattr_pathinfo_cbk;
+                }
+        } else if (!strncmp (name, GF_XATTR_CLRLK_CMD,
+                             strlen (GF_XATTR_CLRLK_CMD))) {
+                if (is_fgetxattr) {
+                        *cbk = afr_fgetxattr_clrlk_cbk;
+                } else {
+                        *cbk = afr_getxattr_clrlk_cbk;
+                }
+        } else if (!strncmp (name, GF_XATTR_LOCKINFO_KEY,
+                             strlen (GF_XATTR_LOCKINFO_KEY))) {
+                if (is_fgetxattr) {
+                        *cbk = afr_fgetxattr_lockinfo_cbk;
+                } else {
+                        *cbk = afr_getxattr_lockinfo_cbk;
+                }
+        } else if (fnmatch (GF_XATTR_STIME_PATTERN, name, FNM_NOESCAPE) == 0) {
+                *cbk = afr_common_getxattr_stime_cbk;
+        } else {
                 is_spl = _gf_false;
+        }
 
 out:
         return is_spl;
@@ -924,14 +1444,14 @@ afr_getxattr_frm_all_children (xlator_t *this, call_frame_t *frame,
                 STACK_WIND_COOKIE (frame, cbk,
                                    (void *) (long) i,
                                    children[i], children[i]->fops->getxattr,
-                                   loc, name);
+                                   loc, name, NULL);
         }
         return;
 }
 
 int32_t
 afr_getxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, const char *name)
+              loc_t *loc, const char *name, dict_t *xdata)
 {
         afr_private_t           *priv         = NULL;
         xlator_t                **children    = NULL;
@@ -944,7 +1464,7 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
         int32_t                 read_child    = -1;
         int                     ret           = -1;
         fop_getxattr_cbk_t      cbk           = NULL;
-
+        int                     afr_xtime_gauge[MCNT_MAX] = {0,};
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -955,6 +1475,8 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
 
         children = priv->children;
 
+        AFR_SBRAIN_CHECK_LOC (loc, out);
+
         AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
@@ -963,38 +1485,98 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
                 goto out;
 
         loc_copy (&local->loc, loc);
-        if (name)
-                local->cont.getxattr.name = gf_strdup (name);
+        if (!name)
+                goto no_name;
+
+        local->cont.getxattr.name = gf_strdup (name);
 
+        if (!strncmp (name, AFR_XATTR_PREFIX,
+                      strlen (AFR_XATTR_PREFIX))) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%s: no data present for key %s",
+                        loc->path, name);
+                op_errno = ENODATA;
+                goto out;
+        }
+        if ((strcmp (GF_XATTR_MARKER_KEY, name) == 0)
+            && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+
+                local->marker.call_count = priv->child_count;
+
+                sub_volumes = alloca ( priv->child_count * sizeof (xlator_t *));
+                for (i = 0, trav = this->children; trav ;
+                     trav = trav->next, i++) {
+
+                        *(sub_volumes + i)  = trav->xlator;
+                }
+
+                if (cluster_getmarkerattr (frame, this, loc, name,
+                                           local, afr_getxattr_unwind,
+                                           sub_volumes,
+                                           priv->child_count,
+                                           MARKER_UUID_TYPE,
+                                           marker_uuid_default_gauge,
+                                           priv->vol_uuid)) {
 
-        if (name) {
-                if (!strncmp (name, AFR_XATTR_PREFIX,
-                              strlen (AFR_XATTR_PREFIX))) {
                         gf_log (this->name, GF_LOG_INFO,
-                                "%s: no data present for key %s",
+                                "%s: failed to get marker attr (%s)",
                                 loc->path, name);
-                        op_errno = ENODATA;
+                        op_errno = EINVAL;
                         goto out;
                 }
-                if ((strcmp (GF_XATTR_MARKER_KEY, name) == 0)
-                    && (-1 == frame->root->pid)) {
 
+                return 0;
+        }
+
+        /*
+         * if we are doing getxattr with pathinfo as the key then we
+         * collect information from all childs
+         */
+        if (afr_is_special_xattr (name, &cbk, 0)) {
+                afr_getxattr_frm_all_children (this, frame, name,
+                                               loc, cbk);
+                return 0;
+        }
+
+        if (XATTR_IS_NODE_UUID (name)) {
+                i = 0;
+                STACK_WIND_COOKIE (frame, afr_getxattr_node_uuid_cbk,
+                                   (void *) (long) i,
+                                   children[i],
+                                   children[i]->fops->getxattr,
+                                   loc, name, xdata);
+                return 0;
+        }
+
+        if (*priv->vol_uuid) {
+                if ((match_uuid_local (name, priv->vol_uuid) == 0)
+                    && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
                         local->marker.call_count = priv->child_count;
 
-                        sub_volumes = alloca ( priv->child_count * sizeof (xlator_t *));
+                        sub_volumes = alloca ( priv->child_count
+                                               * sizeof (xlator_t *));
                         for (i = 0, trav = this->children; trav ;
                              trav = trav->next, i++) {
 
                                 *(sub_volumes + i)  = trav->xlator;
+
                         }
 
-                        if (cluster_getmarkerattr (frame, this, loc, name,
-                                                   local, afr_getxattr_unwind,
+                        /* don't err out on getting ENOTCONN (brick down)
+                         * from a subset of the bricks
+                         */
+                        memcpy (afr_xtime_gauge, marker_xtime_default_gauge,
+                                sizeof (afr_xtime_gauge));
+                        afr_xtime_gauge[MCNT_NOTFOUND] = 0;
+                        afr_xtime_gauge[MCNT_ENOTCONN] = 0;
+                        if (cluster_getmarkerattr (frame, this, loc,
+                                                   name, local,
+                                                   afr_getxattr_unwind,
                                                    sub_volumes,
                                                    priv->child_count,
-                                                   MARKER_UUID_TYPE,
+                                                   MARKER_XTIME_TYPE,
+                                                   afr_xtime_gauge,
                                                    priv->vol_uuid)) {
-
                                 gf_log (this->name, GF_LOG_INFO,
                                         "%s: failed to get marker attr (%s)",
                                         loc->path, name);
@@ -1004,67 +1586,17 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
 
                         return 0;
                 }
-
-                /*
-                 * if we are doing getxattr with pathinfo as the key then we
-                 * collect information from all childs
-                 */
-                if (afr_is_special_xattr (name, &cbk)) {
-                        afr_getxattr_frm_all_children (this, frame, name,
-                                                       loc, cbk);
-                        return 0;
-                }
-
-                if (XATTR_IS_NODE_UUID (name)) {
-                        i = 0;
-                        STACK_WIND_COOKIE (frame, afr_getxattr_node_uuid_cbk,
-                                           (void *) (long) i,
-                                           children[i],
-                                           children[i]->fops->getxattr,
-                                           loc, name);
-                        return 0;
-                }
-
-                if (*priv->vol_uuid) {
-                        if ((match_uuid_local (name, priv->vol_uuid) == 0)
-                            && (-1 == frame->root->pid)) {
-
-                                local->marker.call_count = priv->child_count;
-
-                                sub_volumes = alloca ( priv->child_count * sizeof (xlator_t *));
-                                for (i = 0, trav = this->children; trav ;
-                                     trav = trav->next, i++) {
-
-                                        *(sub_volumes + i)  = trav->xlator;
-
-                                }
-
-                                if (cluster_getmarkerattr (frame, this, loc,
-                                                           name, local,
-                                                           afr_getxattr_unwind,
-                                                           sub_volumes,
-                                                           priv->child_count,
-                                                           MARKER_XTIME_TYPE,
-                                                           priv->vol_uuid)) {
-                                        gf_log (this->name, GF_LOG_INFO,
-                                                "%s: failed to get marker attr (%s)",
-                                                loc->path, name);
-                                        op_errno = EINVAL;
-                                        goto out;
-                                }
-
-                                return 0;
-                        }
-                }
         }
 
+no_name:
         local->fresh_children = afr_children_create (priv->child_count);
         if (!local->fresh_children) {
                 op_errno = ENOMEM;
                 goto out;
         }
 
-        read_child = afr_inode_get_read_ctx (this, loc->inode, local->fresh_children);
+        read_child = afr_inode_get_read_ctx (this, loc->inode,
+                                             local->fresh_children);
         ret = afr_get_call_child (this, local->child_up, read_child,
                                      local->fresh_children,
                                      &call_child,
@@ -1078,12 +1610,12 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
                            (void *) (long) call_child,
                            children[call_child],
                            children[call_child]->fops->getxattr,
-                           loc, name);
+                           loc, name, xdata);
 
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL);
+                AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
@@ -1093,7 +1625,7 @@ out:
 int32_t
 afr_fgetxattr_cbk (call_frame_t *frame, void *cookie,
                    xlator_t *this, int32_t op_ret, int32_t op_errno,
-                   dict_t *dict)
+                   dict_t *dict, dict_t *xdata)
 {
         afr_private_t * priv            = NULL;
         afr_local_t *   local           = NULL;
@@ -1127,7 +1659,8 @@ afr_fgetxattr_cbk (call_frame_t *frame, void *cookie,
                                    children[next_call_child],
                                    children[next_call_child]->fops->fgetxattr,
                                    local->fd,
-                                   local->cont.getxattr.name);
+                                   local->cont.getxattr.name,
+                                   NULL);
         }
 
 out:
@@ -1135,7 +1668,8 @@ out:
                 if (op_ret >= 0 && dict)
                         __filter_xattrs (dict);
 
-                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict);
+                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict,
+                                  xdata);
         }
 
         return 0;
@@ -1143,25 +1677,51 @@ out:
 
 int32_t
 afr_fgetxattr_unwind (call_frame_t *frame,
-                      int op_ret, int op_errno, dict_t *dict)
+                      int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
 
 {
-        AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict);
+        AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
+static void
+afr_fgetxattr_frm_all_children (xlator_t *this, call_frame_t *frame,
+                                const char *name, fd_t *fd,
+                                fop_fgetxattr_cbk_t cbk)
+{
+        afr_private_t   *priv           = NULL;
+        afr_local_t     *local          = NULL;
+        xlator_t        **children      = NULL;
+        int             i               = 0;
+
+        priv     = this->private;
+        children = priv->children;
+
+        local = frame->local;
+        local->call_count = priv->child_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                STACK_WIND_COOKIE (frame, cbk,
+                                   (void *) (long) i,
+                                   children[i], children[i]->fops->fgetxattr,
+                                   fd, name, NULL);
+        }
+
+        return;
+}
+
 int32_t
 afr_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *name)
+               fd_t *fd, const char *name, dict_t *xdata)
 {
-        afr_private_t   *priv         = NULL;
-        xlator_t        **children    = NULL;
-        int             call_child    = 0;
-        afr_local_t     *local        = NULL;
-        int32_t         op_ret        = -1;
-        int32_t         op_errno      = 0;
-        int32_t         read_child    = -1;
-
+        afr_private_t        *priv       = NULL;
+        xlator_t            **children   = NULL;
+        int                   call_child = 0;
+        afr_local_t          *local      = NULL;
+        int32_t               op_ret     = -1;
+        int32_t               op_errno   = 0;
+        int32_t               read_child = -1;
+        fop_fgetxattr_cbk_t   cbk        = NULL;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -1172,6 +1732,8 @@ afr_fgetxattr (call_frame_t *frame, xlator_t *this,
 
         children = priv->children;
 
+        AFR_SBRAIN_CHECK_FD (fd, out);
+
         AFR_LOCAL_ALLOC_OR_GOTO (local, out);
         frame->local = local;
 
@@ -1185,7 +1747,17 @@ afr_fgetxattr (call_frame_t *frame, xlator_t *this,
         if (name)
                 local->cont.getxattr.name = gf_strdup (name);
 
-        /* pathinfo gets handled only in getxattr() */
+        /* pathinfo gets handled only in getxattr(), but we need to handle
+         * lockinfo.
+         * If we are doing fgetxattr with lockinfo as the key then we
+         * collect information from all children.
+         */
+        if (afr_is_special_xattr (name, &cbk, 1)) {
+                afr_fgetxattr_frm_all_children (this, frame, name,
+                                                fd, cbk);
+                return 0;
+        }
+
 
         local->fresh_children = afr_children_create (priv->child_count);
         if (!local->fresh_children) {
@@ -1193,7 +1765,8 @@ afr_fgetxattr (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        read_child = afr_inode_get_read_ctx (this, fd->inode, local->fresh_children);
+        read_child = afr_inode_get_read_ctx (this, fd->inode,
+                                             local->fresh_children);
         op_ret = afr_get_call_child (this, local->child_up, read_child,
                                      local->fresh_children,
                                      &call_child,
@@ -1208,12 +1781,13 @@ afr_fgetxattr (call_frame_t *frame, xlator_t *this,
                            (void *) (long) call_child,
                            children[call_child],
                            children[call_child]->fops->fgetxattr,
-                           fd, name);
+                           fd, name, xdata);
 
         op_ret = 0;
 out:
         if (op_ret == -1) {
-                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, NULL);
+                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, NULL,
+                                  NULL);
         }
         return 0;
 }
@@ -1239,7 +1813,7 @@ int32_t
 afr_readv_cbk (call_frame_t *frame, void *cookie,
                xlator_t *this, int32_t op_ret, int32_t op_errno,
                struct iovec *vector, int32_t count, struct iatt *buf,
-               struct iobref *iobref)
+               struct iobref *iobref, dict_t *xdata)
 {
         afr_private_t * priv            = NULL;
         afr_local_t *   local           = NULL;
@@ -1281,13 +1855,14 @@ afr_readv_cbk (call_frame_t *frame, void *cookie,
                                    children[next_call_child]->fops->readv,
                                    local->fd, local->cont.readv.size,
                                    local->cont.readv.offset,
-                                   local->cont.readv.flags);
+                                   local->cont.readv.flags,
+                                   NULL);
         }
 
 out:
         if (unwind) {
                 AFR_STACK_UNWIND (readv, frame, op_ret, op_errno,
-                                  vector, count, buf, iobref);
+                                  vector, count, buf, iobref, xdata);
         }
 
         return 0;
@@ -1296,7 +1871,7 @@ out:
 
 int32_t
 afr_readv (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, size_t size, off_t offset, uint32_t flags)
+           fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
         afr_private_t * priv       = NULL;
         afr_local_t   * local      = NULL;
@@ -1314,6 +1889,8 @@ afr_readv (call_frame_t *frame, xlator_t *this,
         priv     = this->private;
         children = priv->children;
 
+        AFR_SBRAIN_CHECK_FD (fd, out);
+
         AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
@@ -1343,22 +1920,19 @@ afr_readv (call_frame_t *frame, xlator_t *this,
         local->cont.readv.offset     = offset;
         local->cont.readv.flags      = flags;
 
-        ret = afr_open_fd_fix (frame, this, _gf_false);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
+        afr_open_fd_fix (fd, this);
+
         STACK_WIND_COOKIE (frame, afr_readv_cbk,
                            (void *) (long) call_child,
                            children[call_child],
                            children[call_child]->fops->readv,
-                           fd, size, offset, flags);
+                           fd, size, offset, flags, xdata);
 
         ret = 0;
 out:
         if (ret < 0) {
                 AFR_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL,
-                                  NULL);
+                                  NULL, NULL);
         }
         return 0;
 }
diff --git a/xlators/cluster/afr/src/afr-inode-read.h b/xlators/cluster/afr/src/afr-inode-read.h
index 5ec7411b1..e4091a793 100644
--- a/xlators/cluster/afr/src/afr-inode-read.h
+++ b/xlators/cluster/afr/src/afr-inode-read.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __INODE_READ_H__
@@ -22,30 +13,30 @@
 
 int32_t
 afr_access (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t mask);
+	    loc_t *loc, int32_t mask, dict_t *xdata);
 
 int32_t
 afr_stat (call_frame_t *frame, xlator_t *this,
-	  loc_t *loc);
+	  loc_t *loc, dict_t *xdata);
 
 int32_t
 afr_fstat (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd);
+	   fd_t *fd, dict_t *xdata);
 
 int32_t
 afr_readlink (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, size_t size);
+	      loc_t *loc, size_t size, dict_t *xdata);
 
 int32_t
 afr_readv (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd, size_t size, off_t offset, uint32_t flags);
+	   fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata);
 
 int32_t
 afr_getxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *name);
+	      loc_t *loc, const char *name, dict_t *xdata);
 
 int32_t
 afr_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *name);
+               fd_t *fd, const char *name, dict_t *xdata);
 
 #endif /* __INODE_READ_H__ */
diff --git a/xlators/cluster/afr/src/afr-inode-write.c b/xlators/cluster/afr/src/afr-inode-write.c
index 1d25949db..c1ec69a55 100644
--- a/xlators/cluster/afr/src/afr-inode-write.c
+++ b/xlators/cluster/afr/src/afr-inode-write.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -48,45 +39,151 @@
 #include "afr-transaction.h"
 #include "afr-self-heal-common.h"
 
+void
+__inode_write_fop_cbk (call_frame_t *frame, int child_index, int read_child,
+                       xlator_t *this, int32_t *op_ret, int32_t *op_errno,
+                       struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t     *local = NULL;
+
+        local = frame->local;
+
+        if (afr_fop_failed (*op_ret, *op_errno)) {
+                local->child_errno[child_index] = *op_errno;
+
+                switch (local->op) {
+                case GF_FOP_TRUNCATE:
+                case GF_FOP_FTRUNCATE:
+                        if (*op_errno != EFBIG)
+                                afr_transaction_fop_failed (frame, this,
+                                                            child_index);
+                break;
+                default:
+                        afr_transaction_fop_failed (frame, this, child_index);
+                break;
+                }
+                local->op_errno = *op_errno;
+                goto out;
+        }
+
+        if ((local->success_count == 0) || (read_child == child_index)) {
+                local->op_ret              = *op_ret;
+                if (prebuf)
+                        local->cont.inode_wfop.prebuf  = *prebuf;
+                if (postbuf)
+                        local->cont.inode_wfop.postbuf = *postbuf;
+        }
+
+        local->success_count++;
+out:
+        return;
+}
+
 /* {{{ writev */
 
-int
+void
+afr_writev_copy_outvars (call_frame_t *src_frame, call_frame_t *dst_frame)
+{
+        afr_local_t *src_local = NULL;
+        afr_local_t *dst_local = NULL;
+
+        src_local = src_frame->local;
+        dst_local = dst_frame->local;
+
+        dst_local->op_ret = src_local->op_ret;
+        dst_local->op_errno = src_local->op_errno;
+        dst_local->cont.inode_wfop.prebuf = src_local->cont.inode_wfop.prebuf;
+        dst_local->cont.inode_wfop.postbuf = src_local->cont.inode_wfop.postbuf;
+}
+
+void
 afr_writev_unwind (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+        local = frame->local;
+
+        AFR_STACK_UNWIND (writev, frame,
+                          local->op_ret, local->op_errno,
+                          &local->cont.inode_wfop.prebuf,
+                          &local->cont.inode_wfop.postbuf,
+                          NULL);
+}
+
+call_frame_t*
+afr_transaction_detach_fop_frame (call_frame_t *frame)
+{
+        afr_local_t *   local = NULL;
+        call_frame_t   *fop_frame = NULL;
 
         local = frame->local;
 
         LOCK (&frame->lock);
         {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
+                fop_frame = local->transaction.main_frame;
                 local->transaction.main_frame = NULL;
         }
         UNLOCK (&frame->lock);
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (writev, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  &local->cont.writev.prebuf,
-                                  &local->cont.writev.postbuf);
+        return fop_frame;
+}
+
+int
+afr_transaction_writev_unwind (call_frame_t *frame, xlator_t *this)
+{
+        call_frame_t *fop_frame = NULL;
+
+        fop_frame = afr_transaction_detach_fop_frame (frame);
+
+        if (fop_frame) {
+                afr_writev_copy_outvars (frame, fop_frame);
+                afr_writev_unwind (fop_frame, this);
         }
         return 0;
 }
 
+static void
+afr_writev_handle_short_writes (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t   *local = NULL;
+        afr_private_t *priv  = NULL;
+        int           i      = 0;
+
+        local = frame->local;
+        priv = this->private;
+        /*
+         * We already have the best case result of the writev calls staged
+         * as the return value. Any writev that returns some value less
+         * than the best case is now out of sync, so mark the fop as
+         * failed. Note that fops that have returned with errors have
+         * already been marked as failed.
+         */
+        for (i = 0; i < priv->child_count; i++) {
+                if ((!local->replies[i].valid) ||
+                    (local->replies[i].op_ret == -1))
+                        continue;
+
+                if (local->replies[i].op_ret < local->op_ret)
+                        afr_transaction_fop_failed(frame, this, i);
+        }
+}
 
 int
 afr_writev_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf)
+                     struct iatt *postbuf, dict_t *xdata)
 {
         afr_local_t *   local = NULL;
+        afr_private_t  *priv  = NULL;
+        call_frame_t    *fop_frame = NULL;
         int child_index = (long) cookie;
         int call_count  = -1;
         int read_child  = 0;
+        int      ret = 0;
+        uint32_t open_fd_count = 0;
+        uint32_t write_is_append = 0;
 
         local = frame->local;
+        priv  = this->private;
 
         read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
 
@@ -96,32 +193,81 @@ afr_writev_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->read_child_returned = _gf_true;
                 }
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret              = op_ret;
-                                local->cont.writev.prebuf  = *prebuf;
-                                local->cont.writev.postbuf = *postbuf;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.writev.prebuf  = *prebuf;
-                                local->cont.writev.postbuf = *postbuf;
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, prebuf, postbuf,
+                                       xdata);
+
+		local->replies[child_index].valid = 1;
+		local->replies[child_index].op_ret = op_ret;
+		local->replies[child_index].op_errno = op_errno;
+
+
+		/* stage the best case return value for unwind */
+                if ((local->success_count == 0) || (op_ret > local->op_ret)) {
+                        local->op_ret              = op_ret;
+			local->op_errno		   = op_errno;
+		}
+
+		if (op_ret != -1) {
+                        if (xdata) {
+                                ret = dict_get_uint32 (xdata,
+                                                       GLUSTERFS_OPEN_FD_COUNT,
+                                                       &open_fd_count);
+                                if ((ret == 0) &&
+                                    (open_fd_count > local->open_fd_count)) {
+                                        local->open_fd_count = open_fd_count;
+                                        local->update_open_fd_count = _gf_true;
+                                }
+
+				write_is_append = 0;
+                                ret = dict_get_uint32 (xdata,
+                                                       GLUSTERFS_WRITE_IS_APPEND,
+                                                       &write_is_append);
+                                if (ret || !write_is_append)
+					local->append_write = _gf_false;
                         }
-                }
 
-                local->op_errno = op_errno;
+		}
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
-                local->transaction.unwind (frame, this);
 
-                local->transaction.resume (frame, this);
+                if (local->update_open_fd_count)
+                        afr_handle_open_fd_count (frame, this);
+
+                if (!local->stable_write && !local->append_write)
+			/* An appended write removes the necessity to
+			   fsync() the file. This is because self-heal
+			   has the logic to check for larger file when
+			   the xattrs are not reliably pointing at
+			   a stale file.
+			*/
+                        afr_fd_report_unstable_write (this, local->fd);
+
+                afr_writev_handle_short_writes (frame, this);
+                if (afr_any_fops_failed (local, priv)) {
+                        //Don't unwind until post-op is complete
+                        local->transaction.resume (frame, this);
+                } else {
+                /*
+                 * Generally inode-write fops do transaction.unwind then
+                 * transaction.resume, but writev needs to make sure that
+                 * delayed post-op frame is placed in fdctx before unwind
+                 * happens. This prevents the race of flush doing the
+                 * changelog wakeup first in fuse thread and then this
+                 * writev placing its delayed post-op frame in fdctx.
+                 * This helps flush make sure all the delayed post-ops are
+                 * completed.
+                 */
+
+                        fop_frame = afr_transaction_detach_fop_frame (frame);
+                        afr_writev_copy_outvars (frame, fop_frame);
+                        local->transaction.resume (frame, this);
+                        afr_writev_unwind (fop_frame, this);
+                }
         }
         return 0;
 }
@@ -133,6 +279,8 @@ afr_writev_wind (call_frame_t *frame, xlator_t *this)
         afr_private_t *priv = NULL;
         int i = 0;
         int call_count = -1;
+        dict_t *xdata = NULL;
+        GF_UNUSED int     ret = 0;
 
         local = frame->local;
         priv = this->private;
@@ -146,6 +294,28 @@ afr_writev_wind (call_frame_t *frame, xlator_t *this)
         }
 
         local->call_count = call_count;
+	local->replies = GF_CALLOC(priv->child_count, sizeof(*local->replies),
+				   gf_afr_mt_reply_t);
+	if (!local->replies) {
+		local->op_ret = -1;
+		local->op_errno = ENOMEM;
+		local->transaction.unwind(frame, this);
+		local->transaction.resume(frame, this);
+		return 0;
+	}
+
+        xdata = dict_new ();
+        if (xdata) {
+                ret = dict_set_uint32 (xdata, GLUSTERFS_OPEN_FD_COUNT,
+                                       sizeof (uint32_t));
+		ret = dict_set_uint32 (xdata, GLUSTERFS_WRITE_IS_APPEND,
+				       0);
+		/* Set append_write to be true speculatively. If on any
+		   server it turns not be true, we unset it in the
+		   callback.
+		*/
+		local->append_write = _gf_true;
+        }
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->transaction.pre_op[i]) {
@@ -158,13 +328,17 @@ afr_writev_wind (call_frame_t *frame, xlator_t *this)
                                            local->cont.writev.count,
                                            local->cont.writev.offset,
                                            local->cont.writev.flags,
-                                           local->cont.writev.iobref);
+                                           local->cont.writev.iobref,
+                                           xdata);
 
                         if (!--call_count)
                                 break;
                 }
         }
 
+        if (xdata)
+                dict_unref (xdata);
+
         return 0;
 }
 
@@ -204,7 +378,7 @@ afr_do_writev (call_frame_t *frame, xlator_t *this)
         }
 
         transaction_frame->local = local;
-        frame->local = NULL;
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
 
         local->op = GF_FOP_WRITE;
 
@@ -212,10 +386,17 @@ afr_do_writev (call_frame_t *frame, xlator_t *this)
 
         local->transaction.fop    = afr_writev_wind;
         local->transaction.done   = afr_writev_done;
-        local->transaction.unwind = afr_writev_unwind;
+        local->transaction.unwind = afr_transaction_writev_unwind;
 
         local->transaction.main_frame = frame;
         if (local->fd->flags & O_APPEND) {
+               /*
+                * Backend vfs ignores the 'offset' for append mode fd so
+                * locking just the region provided for the writev does not
+                * give consistency gurantee. The actual write may happen at a
+                * completely different range than the one provided by the
+                * offset, len in the fop. So lock the entire file.
+                */
                 local->transaction.start   = 0;
                 local->transaction.len     = 0;
         } else {
@@ -224,162 +405,91 @@ afr_do_writev (call_frame_t *frame, xlator_t *this)
                                                          local->cont.writev.count);
         }
 
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        op_ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (op_ret < 0) {
+            op_errno = -op_ret;
+            goto out;
+        }
 
         op_ret = 0;
 out:
-        if (op_ret == -1) {
+        if (op_ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (writev, frame, op_ret, op_errno, NULL, NULL);
+                AFR_STACK_UNWIND (writev, frame, op_ret, op_errno, NULL, NULL, NULL);
         }
 
         return 0;
 }
 
-static int
-afr_prepare_loc (call_frame_t *frame, fd_t *fd)
+static void
+afr_trigger_open_fd_self_heal (fd_t *fd, xlator_t *this)
 {
-        afr_local_t    *local = NULL;
-        char           *name = NULL;
-        char           *path = NULL;
-        int             ret = 0;
-
-        if ((!fd) || (!fd->inode))
-                return -1;
-
-        local = frame->local;
-        ret = inode_path (fd->inode, NULL, (char **)&path);
-        if (ret <= 0) {
-                gf_log (frame->this->name, GF_LOG_DEBUG,
-                        "Unable to get path for gfid: %s",
-                        uuid_utoa (fd->inode->gfid));
-                return -1;
-        }
-
-        if (local->loc.path) {
-                if (strcmp (path, local->loc.path))
-                        gf_log (frame->this->name, GF_LOG_DEBUG,
-                                "overwriting old loc->path %s with %s",
-                                local->loc.path, path);
-                GF_FREE ((char *)local->loc.path);
-        }
-        local->loc.path = path;
-
-        name = strrchr (local->loc.path, '/');
-        if (name)
-                name++;
-        local->loc.name = name;
-
-        if (local->loc.inode) {
-                inode_unref (local->loc.inode);
-        }
-        local->loc.inode = inode_ref (fd->inode);
-
-        if (local->loc.parent) {
-                inode_unref (local->loc.parent);
+        call_frame_t    *frame   = NULL;
+        afr_local_t     *local   = NULL;
+        afr_self_heal_t *sh      = NULL;
+        char            *reason  = NULL;
+        int32_t         op_errno = 0;
+        int             ret      = 0;
+
+        if (!fd || !fd->inode || uuid_is_null (fd->inode->gfid)) {
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "Invalid args: "
+                                  "fd: %p, inode: %p", fd,
+                                  fd ? fd->inode : NULL);
+                goto out;
         }
 
-        local->loc.parent = inode_parent (local->loc.inode, 0, NULL);
-
-        return 0;
-}
-
-afr_fd_paused_call_t*
-afr_paused_call_create (call_frame_t *frame)
-{
-        afr_local_t             *local = NULL;
-        afr_fd_paused_call_t   *paused_call = NULL;
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame)
+                goto out;
 
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
-        GF_ASSERT (local->fop_call_continue);
-
-        paused_call = GF_CALLOC (1, sizeof (*paused_call),
-                                  gf_afr_fd_paused_call_t);
-        if (paused_call) {
-                INIT_LIST_HEAD (&paused_call->call_list);
-                paused_call->frame = frame;
-        }
-
-        return paused_call;
-}
-
-static int
-afr_pause_fd_fop (call_frame_t *frame, xlator_t *this, afr_fd_ctx_t *fd_ctx)
-{
-        afr_fd_paused_call_t *paused_call = NULL;
-        int                    ret = 0;
-
-        paused_call = afr_paused_call_create (frame);
-        if (paused_call)
-                list_add (&paused_call->call_list, &fd_ctx->paused_calls);
-        else
-                ret = -ENOMEM;
-
-        return ret;
-}
+        ret = afr_local_init (local, this->private, &op_errno);
+        if (ret < 0)
+                goto out;
 
-static void
-afr_trigger_open_fd_self_heal (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t             *local = NULL;
-        afr_self_heal_t         *sh = NULL;
-        inode_t                 *inode = NULL;
-        char                    *reason = NULL;
+        local->loc.inode = inode_ref (fd->inode);
+        ret = loc_path (&local->loc, NULL);
+        if (ret < 0)
+                goto out;
 
-        local = frame->local;
         sh    = &local->self_heal;
-        inode = local->fd->inode;
-
-        sh->do_missing_entry_self_heal = _gf_true;
-        sh->do_gfid_self_heal = _gf_true;
-        sh->do_data_self_heal = _gf_true;
+        sh->do_metadata_self_heal = _gf_true;
+        if (fd->inode->ia_type == IA_IFREG)
+                sh->do_data_self_heal = _gf_true;
+        else if (fd->inode->ia_type == IA_IFDIR)
+                sh->do_entry_self_heal = _gf_true;
 
         reason = "subvolume came online";
-        afr_launch_self_heal (frame, this, inode, _gf_true, inode->ia_type,
-                              reason, NULL, NULL);
+        afr_launch_self_heal (frame, this, fd->inode, _gf_true,
+                              fd->inode->ia_type, reason, NULL, NULL);
+        return;
+out:
+        AFR_STACK_DESTROY (frame);
 }
 
-int
-afr_open_fd_fix (call_frame_t *frame, xlator_t *this, gf_boolean_t pause_fop)
-{
-        int                     ret = 0;
-        int                     i   = 0;
-        afr_fd_ctx_t            *fd_ctx = NULL;
-        gf_boolean_t            need_self_heal = _gf_false;
-        int                     *need_open = NULL;
-        int                     need_open_count = 0;
-        afr_local_t             *local = NULL;
-        afr_private_t           *priv = NULL;
-        gf_boolean_t            fop_continue = _gf_true;
+void
+afr_open_fd_fix (fd_t *fd, xlator_t *this)
+{
+        int           ret             = 0;
+        int           i               = 0;
+        afr_fd_ctx_t  *fd_ctx         = NULL;
+        gf_boolean_t  need_self_heal  = _gf_false;
+        int           *need_open      = NULL;
+        size_t        need_open_count = 0;
+        afr_private_t *priv           = NULL;
 
-        local = frame->local;
         priv  = this->private;
 
-        GF_ASSERT (local->fd);
-
-        if (fd_is_anonymous (local->fd)) {
-                fop_continue = _gf_true;
-                goto out;
-        }
-
-        if (pause_fop)
-                GF_ASSERT (local->fop_call_continue);
-
-        ret = afr_prepare_loc (frame, local->fd);
-        if (ret < 0) {
-                //File does not exist we cant open it.
-                ret = 0;
+        if (!afr_is_fd_fixable (fd))
                 goto out;
-        }
 
-        fd_ctx = afr_fd_ctx_get (local->fd, this);
-        if (!fd_ctx) {
-                ret = -EINVAL;
+        fd_ctx = afr_fd_ctx_get (fd, this);
+        if (!fd_ctx)
                 goto out;
-        }
 
-        LOCK (&local->fd->lock);
+        LOCK (&fd->lock);
         {
                 if (fd_ctx->up_count < priv->up_count) {
                         need_self_heal = _gf_true;
@@ -387,62 +497,40 @@ afr_open_fd_fix (call_frame_t *frame, xlator_t *this, gf_boolean_t pause_fop)
                         fd_ctx->down_count = priv->down_count;
                 }
 
+                need_open = alloca (priv->child_count * sizeof (*need_open));
                 for (i = 0; i < priv->child_count; i++) {
-                        if ((fd_ctx->opened_on[i] == AFR_FD_NOT_OPENED) &&
-                            local->child_up[i]) {
-                                fd_ctx->opened_on[i] = AFR_FD_OPENING;
-                                if (!need_open)
-                                        need_open = GF_CALLOC (priv->child_count,
-                                                               sizeof (*need_open),
-                                                               gf_afr_mt_int32_t);
-                                need_open[i] = 1;
-                                need_open_count++;
-                        } else if (pause_fop && local->child_up[i] &&
-                                   (fd_ctx->opened_on[i] == AFR_FD_OPENING)) {
-                                local->fop_paused = _gf_true;
-                        }
-                }
+                        need_open[i] = 0;
+                        if (fd_ctx->opened_on[i] != AFR_FD_NOT_OPENED)
+                                continue;
+
+                        if (!priv->child_up[i])
+                                continue;
 
-                if (local->fop_paused) {
-                        GF_ASSERT (pause_fop);
-                        gf_log (this->name, GF_LOG_INFO, "Pause fd %p",
-                                local->fd);
-                        ret = afr_pause_fd_fop (frame, this, fd_ctx);
-                        if (ret)
-                                goto unlock;
-                        fop_continue = _gf_false;
+                        fd_ctx->opened_on[i] = AFR_FD_OPENING;
+
+                        need_open[i] = 1;
+                        need_open_count++;
                 }
         }
-unlock:
-        UNLOCK (&local->fd->lock);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to fix fd for %s",
-                        local->loc.path);
-                fop_continue = _gf_false;
+        UNLOCK (&fd->lock);
+        if (ret)
                 goto out;
-        }
 
         if (need_self_heal)
-                afr_trigger_open_fd_self_heal (frame, this);
+                afr_trigger_open_fd_self_heal (fd, this);
 
         if (!need_open_count)
                 goto out;
 
-        gf_log (this->name, GF_LOG_INFO, "Opening fd %p", local->fd);
-        afr_fix_open (frame, this, fd_ctx, need_open_count, need_open);
-        fop_continue = _gf_false;
+        afr_fix_open (this, fd, need_open_count, need_open);
 out:
-        if (need_open)
-                GF_FREE (need_open);
-        if (fop_continue && local->fop_call_continue)
-                local->fop_call_continue (frame, this);
-        return ret;
+        return;
 }
 
 int
 afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
             struct iovec *vector, int32_t count, off_t offset,
-            uint32_t flags, struct iobref *iobref)
+            uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         afr_private_t * priv  = NULL;
         afr_local_t   * local = NULL;
@@ -455,6 +543,11 @@ afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         priv = this->private;
 
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
+
         QUORUM_CHECK(writev,out);
 
         AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
@@ -471,18 +564,20 @@ afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         local->cont.writev.iobref     = iobref_ref (iobref);
 
         local->fd                = fd_ref (fd);
-        local->fop_call_continue = afr_do_writev;
 
-        ret = afr_open_fd_fix (frame, this, _gf_true);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
+	/* detect here, but set it in writev_wind_cbk *after* the unstable
+	   write is performed
+	*/
+	local->stable_write = !!((fd->flags|flags)&(O_SYNC|O_DSYNC));
+
+        afr_open_fd_fix (fd, this);
+
+        afr_do_writev (frame, this);
 
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL);
+                AFR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -511,8 +606,9 @@ afr_truncate_unwind (call_frame_t *frame, xlator_t *this)
         if (main_frame) {
                 AFR_STACK_UNWIND (truncate, main_frame, local->op_ret,
                                   local->op_errno,
-                                  &local->cont.truncate.prebuf,
-                                  &local->cont.truncate.postbuf);
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
+                                  NULL);
         }
 
         return 0;
@@ -522,17 +618,14 @@ afr_truncate_unwind (call_frame_t *frame, xlator_t *this)
 int
 afr_truncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf)
+                       struct iatt *postbuf, dict_t *xdata)
 {
         afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
         int child_index = (long) cookie;
         int read_child  = 0;
         int call_count  = -1;
-        int need_unwind = 0;
 
         local = frame->local;
-        priv  = this->private;
 
         read_child = afr_inode_get_read_ctx (this, local->loc.inode, NULL);
 
@@ -542,38 +635,22 @@ afr_truncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->read_child_returned = _gf_true;
                 }
 
-                if (afr_fop_failed (op_ret, op_errno) && op_errno != EFBIG)
-                        afr_transaction_fop_failed (frame, this, child_index);
-
                 if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.truncate.prebuf  = *prebuf;
-                                local->cont.truncate.postbuf = *postbuf;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.truncate.prebuf  = *prebuf;
-                                local->cont.truncate.postbuf = *postbuf;
-                        }
-
-                        local->success_count++;
-
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
+			if (prebuf->ia_size != postbuf->ia_size)
+				local->stable_write = _gf_false;
                 }
-                local->op_errno = op_errno;
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, prebuf, postbuf,
+                                       xdata);
         }
         UNLOCK (&frame->lock);
 
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
+		if (local->stable_write && afr_txn_nothing_failed (frame, this))
+			local->transaction.unwind (frame, this);
+
                 local->transaction.resume (frame, this);
         }
 
@@ -601,6 +678,7 @@ afr_truncate_wind (call_frame_t *frame, xlator_t *this)
         }
 
         local->call_count = call_count;
+	local->stable_write = _gf_true;
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->transaction.pre_op[i]) {
@@ -609,7 +687,8 @@ afr_truncate_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i],
                                            priv->children[i]->fops->truncate,
                                            &local->loc,
-                                           local->cont.truncate.offset);
+                                           local->cont.truncate.offset,
+                                           NULL);
 
                         if (!--call_count)
                                 break;
@@ -637,7 +716,7 @@ afr_truncate_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_truncate (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, off_t offset)
+              loc_t *loc, off_t offset, dict_t *xdata)
 {
         afr_private_t * priv  = NULL;
         afr_local_t   * local = NULL;
@@ -678,14 +757,18 @@ afr_truncate (call_frame_t *frame, xlator_t *this,
         local->transaction.start   = offset;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
         if (ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL);
+                AFR_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
         }
 
         return 0;
@@ -716,8 +799,9 @@ afr_ftruncate_unwind (call_frame_t *frame, xlator_t *this)
         if (main_frame) {
                 AFR_STACK_UNWIND (ftruncate, main_frame, local->op_ret,
                                   local->op_errno,
-                                  &local->cont.ftruncate.prebuf,
-                                  &local->cont.ftruncate.postbuf);
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
+                                  NULL);
         }
         return 0;
 }
@@ -726,17 +810,14 @@ afr_ftruncate_unwind (call_frame_t *frame, xlator_t *this)
 int
 afr_ftruncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf)
+                        struct iatt *postbuf, dict_t *xdata)
 {
         afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
         int child_index = (long) cookie;
         int call_count  = -1;
-        int need_unwind = 0;
         int read_child  = 0;
 
         local = frame->local;
-        priv  = this->private;
 
         read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
 
@@ -746,38 +827,22 @@ afr_ftruncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->read_child_returned = _gf_true;
                 }
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
                 if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.ftruncate.prebuf  = *prebuf;
-                                local->cont.ftruncate.postbuf = *postbuf;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.ftruncate.prebuf  = *prebuf;
-                                local->cont.ftruncate.postbuf = *postbuf;
-                        }
-
-                        local->success_count++;
-
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
+			if (prebuf->ia_size != postbuf->ia_size)
+				local->stable_write = _gf_false;
                 }
-                local->op_errno = op_errno;
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, prebuf, postbuf,
+                                       xdata);
         }
         UNLOCK (&frame->lock);
 
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
+		if (local->stable_write && afr_txn_nothing_failed (frame, this))
+			local->transaction.unwind (frame, this);
+
                 local->transaction.resume (frame, this);
         }
 
@@ -805,6 +870,7 @@ afr_ftruncate_wind (call_frame_t *frame, xlator_t *this)
         }
 
         local->call_count = call_count;
+	local->stable_write = _gf_true;
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->transaction.pre_op[i]) {
@@ -812,7 +878,9 @@ afr_ftruncate_wind (call_frame_t *frame, xlator_t *this)
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->ftruncate,
-                                           local->fd, local->cont.ftruncate.offset);
+                                           local->fd,
+                                           local->cont.ftruncate.offset,
+                                           NULL);
 
                         if (!--call_count)
                                 break;
@@ -867,14 +935,19 @@ afr_do_ftruncate (call_frame_t *frame, xlator_t *this)
         local->transaction.start   = local->cont.ftruncate.offset;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        op_ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (op_ret < 0) {
+            op_errno = -op_ret;
+            goto out;
+        }
 
         op_ret = 0;
 out:
-        if (op_ret == -1) {
+        if (op_ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, NULL, NULL);
+                AFR_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, NULL,
+                                  NULL, NULL);
         }
 
         return 0;
@@ -883,7 +956,7 @@ out:
 
 int
 afr_ftruncate (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, off_t offset)
+               fd_t *fd, off_t offset, dict_t *xdata)
 {
         afr_private_t * priv  = NULL;
         afr_local_t   * local = NULL;
@@ -897,6 +970,10 @@ afr_ftruncate (call_frame_t *frame, xlator_t *this,
 
         priv = this->private;
 
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
         QUORUM_CHECK(ftruncate,out);
 
         AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
@@ -909,20 +986,17 @@ afr_ftruncate (call_frame_t *frame, xlator_t *this,
         local->cont.ftruncate.offset  = offset;
 
         local->fd = fd_ref (fd);
-        local->fop_call_continue = afr_do_ftruncate;
 
-        ret = afr_open_fd_fix (frame, this, _gf_true);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
+        afr_open_fd_fix (fd, this);
+
+        afr_do_ftruncate (frame, this);
 
         ret = 0;
 out:
         if (ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL);
+                AFR_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
         }
 
         return 0;
@@ -951,8 +1025,9 @@ afr_setattr_unwind (call_frame_t *frame, xlator_t *this)
         if (main_frame) {
                 AFR_STACK_UNWIND (setattr, main_frame, local->op_ret,
                                   local->op_errno,
-                                  &local->cont.setattr.preop_buf,
-                                  &local->cont.setattr.postop_buf);
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
+                                  NULL);
         }
 
         return 0;
@@ -962,7 +1037,7 @@ afr_setattr_unwind (call_frame_t *frame, xlator_t *this)
 int
 afr_setattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno,
-                      struct iatt *preop, struct iatt *postop)
+                      struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
         afr_local_t *   local = NULL;
         afr_private_t * priv  = NULL;
@@ -982,29 +1057,14 @@ afr_setattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->read_child_returned = _gf_true;
                 }
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.setattr.preop_buf  = *preop;
-                                local->cont.setattr.postop_buf = *postop;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.setattr.preop_buf  = *preop;
-                                local->cont.setattr.postop_buf = *postop;
-                        }
-
-                        local->success_count++;
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, preop, postop,
+                                       xdata);
 
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
+                if ((local->success_count >= priv->wait_count)
+                    && local->read_child_returned) {
+                        need_unwind = 1;
                 }
-                local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
@@ -1050,7 +1110,8 @@ afr_setattr_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i]->fops->setattr,
                                            &local->loc,
                                            &local->cont.setattr.in_buf,
-                                           local->cont.setattr.valid);
+                                           local->cont.setattr.valid,
+                                           NULL);
 
                         if (!--call_count)
                                 break;
@@ -1078,7 +1139,7 @@ afr_setattr_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_setattr (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, struct iatt *buf, int32_t valid)
+             loc_t *loc, struct iatt *buf, int32_t valid, dict_t *xdata)
 {
         afr_private_t * priv  = NULL;
         afr_local_t   * local = NULL;
@@ -1120,14 +1181,18 @@ afr_setattr (call_frame_t *frame, xlator_t *this,
         local->transaction.start   = LLONG_MAX - 1;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
         if (ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL);
+                AFR_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
         }
 
         return 0;
@@ -1154,8 +1219,9 @@ afr_fsetattr_unwind (call_frame_t *frame, xlator_t *this)
         if (main_frame) {
                 AFR_STACK_UNWIND (fsetattr, main_frame, local->op_ret,
                                   local->op_errno,
-                                  &local->cont.fsetattr.preop_buf,
-                                  &local->cont.fsetattr.postop_buf);
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
+                                  NULL);
         }
 
         return 0;
@@ -1165,7 +1231,7 @@ afr_fsetattr_unwind (call_frame_t *frame, xlator_t *this)
 int
 afr_fsetattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno,
-                       struct iatt *preop, struct iatt *postop)
+                       struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
         afr_local_t *   local = NULL;
         afr_private_t * priv  = NULL;
@@ -1185,29 +1251,14 @@ afr_fsetattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->read_child_returned = _gf_true;
                 }
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, preop, postop,
+                                       xdata);
 
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.fsetattr.preop_buf  = *preop;
-                                local->cont.fsetattr.postop_buf = *postop;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.fsetattr.preop_buf  = *preop;
-                                local->cont.fsetattr.postop_buf = *postop;
-                        }
-
-                        local->success_count++;
-
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
+                if ((local->success_count >= priv->wait_count)
+                    && local->read_child_returned) {
+                        need_unwind = 1;
                 }
-                local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
@@ -1253,7 +1304,8 @@ afr_fsetattr_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i]->fops->fsetattr,
                                            local->fd,
                                            &local->cont.fsetattr.in_buf,
-                                           local->cont.fsetattr.valid);
+                                           local->cont.fsetattr.valid,
+                                           NULL);
 
                         if (!--call_count)
                                 break;
@@ -1280,7 +1332,7 @@ afr_fsetattr_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_fsetattr (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, struct iatt *buf, int32_t valid)
+              fd_t *fd, struct iatt *buf, int32_t valid, dict_t *xdata)
 {
         afr_private_t * priv  = NULL;
         afr_local_t   * local = NULL;
@@ -1294,6 +1346,11 @@ afr_fsetattr (call_frame_t *frame, xlator_t *this,
 
         priv = this->private;
 
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
+
         QUORUM_CHECK(fsetattr,out);
 
         transaction_frame = copy_frame (frame);
@@ -1318,24 +1375,24 @@ afr_fsetattr (call_frame_t *frame, xlator_t *this,
 
         local->fd                 = fd_ref (fd);
 
-        ret = afr_open_fd_fix (transaction_frame, this, _gf_false);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
+        afr_open_fd_fix (fd, this);
 
         local->transaction.main_frame = frame;
         local->transaction.start   = LLONG_MAX - 1;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
         if (ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL);
+                AFR_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
         }
 
         return 0;
@@ -1363,7 +1420,8 @@ afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
 
         if (main_frame) {
                 AFR_STACK_UNWIND (setxattr, main_frame,
-                                  local->op_ret, local->op_errno);
+                                  local->op_ret, local->op_errno,
+                                  NULL);
         }
         return 0;
 }
@@ -1371,30 +1429,25 @@ afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
 
 int
 afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t      *local         = NULL;
-        afr_private_t    *priv          = NULL;
-        int               call_count    = -1;
-        int               need_unwind   = 0;
+        afr_local_t   *local      = NULL;
+        afr_private_t *priv       = NULL;
+        int           call_count  = -1;
+        int           need_unwind = 0;
+        int           child_index = (long) cookie;
 
         local = frame->local;
         priv = this->private;
 
         LOCK (&frame->lock);
         {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->child_count) {
-                                need_unwind = 1;
-                        }
+                __inode_write_fop_cbk (frame, child_index, -1, this,
+                                       &op_ret, &op_errno, NULL, NULL,
+                                       xdata);
+                if (local->success_count == priv->child_count) {
+                        need_unwind = 1;
                 }
-
-                local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
@@ -1440,7 +1493,8 @@ afr_setxattr_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i]->fops->setxattr,
                                            &local->loc,
                                            local->cont.setxattr.dict,
-                                           local->cont.setxattr.flags);
+                                           local->cont.setxattr.flags,
+                                           NULL);
 
                         if (!--call_count)
                                 break;
@@ -1465,22 +1519,21 @@ afr_setxattr_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *dict, int32_t flags)
+              loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata)
 {
         afr_private_t  *priv              = NULL;
         afr_local_t    *local             = NULL;
         call_frame_t   *transaction_frame = NULL;
-        data_pair_t    *trav              = NULL;
         int             ret               = -1;
         int             op_errno          = EINVAL;
 
         VALIDATE_OR_GOTO (this, out);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
-                                   trav, op_errno, out);
+                                   op_errno, out);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
-                                   trav, op_errno, out);
+                                   op_errno, out);
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this->private, out);
@@ -1514,14 +1567,18 @@ afr_setxattr (call_frame_t *frame, xlator_t *this,
         local->transaction.start   = LLONG_MAX - 1;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
         if (ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (setxattr, frame, -1, op_errno);
+                AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
         }
 
         return 0;
@@ -1548,7 +1605,8 @@ afr_fsetxattr_unwind (call_frame_t *frame, xlator_t *this)
 
         if (main_frame) {
                 AFR_STACK_UNWIND (fsetxattr, main_frame,
-                                  local->op_ret, local->op_errno);
+                                  local->op_ret, local->op_errno,
+                                  NULL);
         }
         return 0;
 }
@@ -1556,30 +1614,26 @@ afr_fsetxattr_unwind (call_frame_t *frame, xlator_t *this)
 
 int
 afr_fsetxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t     *local       = NULL;
-        afr_private_t   *priv        = NULL;
-        int              call_count  = -1;
-        int              need_unwind = 0;
+        afr_local_t   *local      = NULL;
+        afr_private_t *priv       = NULL;
+        int           call_count  = -1;
+        int           need_unwind = 0;
+        int           child_index = (long) cookie;
 
         local = frame->local;
         priv = this->private;
 
         LOCK (&frame->lock);
         {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
 
-                        if (local->success_count == priv->child_count) {
-                                need_unwind = 1;
-                        }
+                __inode_write_fop_cbk (frame, child_index, -1, this,
+                                       &op_ret, &op_errno, NULL, NULL,
+                                       xdata);
+                if (local->success_count == priv->child_count) {
+                        need_unwind = 1;
                 }
-
-                local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
@@ -1625,7 +1679,8 @@ afr_fsetxattr_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i]->fops->fsetxattr,
                                            local->fd,
                                            local->cont.fsetxattr.dict,
-                                           local->cont.fsetxattr.flags);
+                                           local->cont.fsetxattr.flags,
+                                           NULL);
 
                         if (!--call_count)
                                 break;
@@ -1650,30 +1705,31 @@ afr_fsetxattr_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_fsetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, dict_t *dict, int32_t flags)
+               fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata)
 {
         afr_private_t    *priv              = NULL;
         afr_local_t      *local             = NULL;
         call_frame_t     *transaction_frame = NULL;
         int               ret               = -1;
         int               op_errno          = EINVAL;
-        data_pair_t      *trav              = NULL;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
         VALIDATE_OR_GOTO (this->private, out);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
-                                   trav, op_errno, out);
+                                   op_errno, out);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
-                                   trav, op_errno, out);
-
-        if (ret)
-                goto out;
+                                   op_errno, out);
 
         priv = this->private;
 
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
+
         QUORUM_CHECK(fsetxattr,out);
 
         AFR_LOCAL_ALLOC_OR_GOTO (local, out);
@@ -1704,14 +1760,18 @@ afr_fsetxattr (call_frame_t *frame, xlator_t *this,
         local->transaction.start  = LLONG_MAX - 1;
         local->transaction.len    = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
         if (ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (fsetxattr, frame, -1, op_errno);
+                AFR_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
         }
 
         return 0;
@@ -1741,7 +1801,8 @@ afr_removexattr_unwind (call_frame_t *frame, xlator_t *this)
 
         if (main_frame) {
                 AFR_STACK_UNWIND (removexattr, main_frame,
-                                  local->op_ret, local->op_errno);
+                                  local->op_ret, local->op_errno,
+                                  NULL);
         }
         return 0;
 }
@@ -1749,30 +1810,25 @@ afr_removexattr_unwind (call_frame_t *frame, xlator_t *this)
 
 int
 afr_removexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int call_count  = -1;
-        int need_unwind = 0;
+        afr_local_t   *local      = NULL;
+        afr_private_t *priv       = NULL;
+        int           call_count  = -1;
+        int           need_unwind = 0;
+        int           child_index = (long) cookie;
 
         local = frame->local;
         priv = this->private;
 
         LOCK (&frame->lock);
         {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->wait_count) {
-                                need_unwind = 1;
-                        }
+                __inode_write_fop_cbk (frame, child_index, -1, this,
+                                       &op_ret, &op_errno, NULL, NULL,
+                                       xdata);
+                if (local->success_count == priv->wait_count) {
+                        need_unwind = 1;
                 }
-
-                local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
@@ -1817,7 +1873,8 @@ afr_removexattr_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i],
                                            priv->children[i]->fops->removexattr,
                                            &local->loc,
-                                           local->cont.removexattr.name);
+                                           local->cont.removexattr.name,
+                                           NULL);
 
                         if (!--call_count)
                                 break;
@@ -1843,7 +1900,7 @@ afr_removexattr_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_removexattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *name)
+                 loc_t *loc, const char *name, dict_t *xdata)
 {
         afr_private_t   *priv              = NULL;
         afr_local_t     *local             = NULL;
@@ -1892,14 +1949,18 @@ afr_removexattr (call_frame_t *frame, xlator_t *this,
         local->transaction.start   = LLONG_MAX - 1;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
         ret = 0;
 out:
         if (ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (removexattr, frame, -1, op_errno);
+                AFR_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
         }
 
         return 0;
@@ -1924,7 +1985,8 @@ afr_fremovexattr_unwind (call_frame_t *frame, xlator_t *this)
 
         if (main_frame) {
                 AFR_STACK_UNWIND (fremovexattr, main_frame,
-                                  local->op_ret, local->op_errno);
+                                  local->op_ret, local->op_errno,
+                                  NULL);
         }
         return 0;
 }
@@ -1932,30 +1994,26 @@ afr_fremovexattr_unwind (call_frame_t *frame, xlator_t *this)
 
 int
 afr_fremovexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int call_count  = -1;
-        int need_unwind = 0;
+        afr_local_t *   local       = NULL;
+        afr_private_t * priv        = NULL;
+        int             call_count  = -1;
+        int             need_unwind = 0;
+        int             child_index = (long) cookie;
 
         local = frame->local;
         priv = this->private;
 
         LOCK (&frame->lock);
         {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
+                __inode_write_fop_cbk (frame, child_index, -1, this,
+                                       &op_ret, &op_errno, NULL, NULL,
+                                       xdata);
 
-                        if (local->success_count == priv->wait_count) {
-                                need_unwind = 1;
-                        }
+                if (local->success_count == priv->wait_count) {
+                        need_unwind = 1;
                 }
-
-                local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
@@ -2000,7 +2058,8 @@ afr_fremovexattr_wind (call_frame_t *frame, xlator_t *this)
                                            priv->children[i],
                                            priv->children[i]->fops->fremovexattr,
                                            local->fd,
-                                           local->cont.removexattr.name);
+                                           local->cont.removexattr.name,
+                                           NULL);
 
                         if (!--call_count)
                                 break;
@@ -2026,7 +2085,7 @@ afr_fremovexattr_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_fremovexattr (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, const char *name)
+                  fd_t *fd, const char *name, dict_t *xdata)
 {
         afr_private_t * priv  = NULL;
         afr_local_t   * local = NULL;
@@ -2047,6 +2106,10 @@ afr_fremovexattr (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (this->private, out);
 
         priv = this->private;
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
 
         QUORUM_CHECK(fremovexattr, out);
 
@@ -2079,15 +2142,720 @@ afr_fremovexattr (call_frame_t *frame, xlator_t *this,
         local->transaction.start   = LLONG_MAX - 1;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        op_ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (op_ret < 0) {
+            op_errno = -op_ret;
+            goto out;
+        }
 
         op_ret = 0;
 out:
-        if (op_ret == -1) {
+        if (op_ret < 0) {
                 if (transaction_frame)
                         AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno);
+                AFR_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, NULL);
+        }
+
+        return 0;
+}
+
+static int
+afr_fallocate_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame)
+                        main_frame = local->transaction.main_frame;
+                local->transaction.main_frame = NULL;
         }
+        UNLOCK (&frame->lock);
 
+        if (main_frame) {
+                AFR_STACK_UNWIND (fallocate, main_frame, local->op_ret,
+                                  local->op_errno,
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
+                                  NULL);
+        }
         return 0;
 }
+
+static int
+afr_fallocate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t *   local = NULL;
+        afr_private_t * priv  = NULL;
+        int child_index = (long) cookie;
+        int call_count  = -1;
+        int need_unwind = 0;
+        int read_child  = 0;
+
+        local = frame->local;
+        priv  = this->private;
+
+        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
+
+        LOCK (&frame->lock);
+        {
+                if (child_index == read_child) {
+                        local->read_child_returned = _gf_true;
+                }
+
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, prebuf, postbuf,
+                                       xdata);
+
+                if ((local->success_count >= priv->wait_count)
+                    && local->read_child_returned) {
+                        need_unwind = 1;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (need_unwind)
+                local->transaction.unwind (frame, this);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+        }
+
+        return 0;
+}
+
+static int
+afr_fallocate_wind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_fallocate_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->fallocate,
+                                           local->fd,
+                                           local->cont.fallocate.mode,
+                                           local->cont.fallocate.offset,
+                                           local->cont.fallocate.len,
+                                           NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
+}
+
+static int
+afr_fallocate_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+
+        local = frame->local;
+
+        local->transaction.unwind (frame, this);
+
+        AFR_STACK_DESTROY (frame);
+
+        return 0;
+}
+
+static int
+afr_do_fallocate (call_frame_t *frame, xlator_t *this)
+{
+        call_frame_t * transaction_frame = NULL;
+        afr_local_t *  local             = NULL;
+        int op_ret   = -1;
+        int op_errno = 0;
+
+        local = frame->local;
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                goto out;
+        }
+
+        transaction_frame->local = local;
+        frame->local = NULL;
+
+        local->op = GF_FOP_FALLOCATE;
+
+        local->transaction.fop    = afr_fallocate_wind;
+        local->transaction.done   = afr_fallocate_done;
+        local->transaction.unwind = afr_fallocate_unwind;
+
+        local->transaction.main_frame = frame;
+
+        local->transaction.start   = local->cont.fallocate.offset;
+        local->transaction.len     = 0;
+
+        /* fallocate can modify the file size */
+        op_ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (op_ret < 0) {
+            op_errno = -op_ret;
+            goto out;
+        }
+
+        op_ret = 0;
+out:
+        if (op_ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (fallocate, frame, op_ret, op_errno, NULL,
+                                  NULL, NULL);
+        }
+
+        return 0;
+}
+
+int
+afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+               off_t offset, size_t len, dict_t *xdata)
+{
+        afr_private_t * priv  = NULL;
+        afr_local_t   * local = NULL;
+        call_frame_t   *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
+        QUORUM_CHECK(fallocate,out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        local->cont.fallocate.mode = mode;
+        local->cont.fallocate.offset  = offset;
+        local->cont.fallocate.len = len;
+
+        local->fd = fd_ref (fd);
+
+        afr_open_fd_fix (fd, this);
+
+        afr_do_fallocate (frame, this);
+
+        ret = 0;
+out:
+        if (ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+        }
+
+        return 0;
+}
+
+/* }}} */
+
+/* {{{ discard */
+
+static int
+afr_discard_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame)
+                        main_frame = local->transaction.main_frame;
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (discard, main_frame, local->op_ret,
+                                  local->op_errno,
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
+                                  NULL);
+        }
+        return 0;
+}
+
+static int
+afr_discard_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t *   local = NULL;
+        afr_private_t * priv  = NULL;
+        int child_index = (long) cookie;
+        int call_count  = -1;
+        int need_unwind = 0;
+        int read_child  = 0;
+
+        local = frame->local;
+        priv  = this->private;
+
+        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
+
+        LOCK (&frame->lock);
+        {
+                if (child_index == read_child) {
+                        local->read_child_returned = _gf_true;
+                }
+
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, prebuf, postbuf,
+                                       xdata);
+
+                if ((local->success_count >= priv->wait_count)
+                    && local->read_child_returned) {
+                        need_unwind = 1;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (need_unwind)
+                local->transaction.unwind (frame, this);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+        }
+
+        return 0;
+}
+
+static int
+afr_discard_wind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_discard_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->discard,
+                                           local->fd,
+                                           local->cont.discard.offset,
+                                           local->cont.discard.len,
+                                           NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
+}
+
+static int
+afr_discard_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+
+        local = frame->local;
+
+        local->transaction.unwind (frame, this);
+
+        AFR_STACK_DESTROY (frame);
+
+        return 0;
+}
+
+static int
+afr_do_discard (call_frame_t *frame, xlator_t *this)
+{
+        call_frame_t * transaction_frame = NULL;
+        afr_local_t *  local             = NULL;
+        int op_ret   = -1;
+        int op_errno = 0;
+
+        local = frame->local;
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                goto out;
+        }
+
+        transaction_frame->local = local;
+        frame->local = NULL;
+
+        local->op = GF_FOP_DISCARD;
+
+        local->transaction.fop    = afr_discard_wind;
+        local->transaction.done   = afr_discard_done;
+        local->transaction.unwind = afr_discard_unwind;
+
+        local->transaction.main_frame = frame;
+
+        local->transaction.start   = local->cont.discard.offset;
+        local->transaction.len     = 0;
+
+        op_ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (op_ret < 0) {
+            op_errno = -op_ret;
+            goto out;
+        }
+
+        op_ret = 0;
+out:
+        if (op_ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (discard, frame, op_ret, op_errno, NULL,
+                                  NULL, NULL);
+        }
+
+        return 0;
+}
+
+int
+afr_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata)
+{
+        afr_private_t * priv  = NULL;
+        afr_local_t   * local = NULL;
+        call_frame_t   *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
+        QUORUM_CHECK(discard, out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        local->cont.discard.offset  = offset;
+        local->cont.discard.len = len;
+
+        local->fd = fd_ref (fd);
+
+        afr_open_fd_fix (fd, this);
+
+        afr_do_discard(frame, this);
+
+        ret = 0;
+out:
+        if (ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
+        }
+
+        return 0;
+}
+
+
+/* {{{ zerofill */
+
+static int
+afr_zerofill_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local            = NULL;
+        call_frame_t    *main_frame       = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame) {
+                        main_frame = local->transaction.main_frame;
+                }
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (zerofill, main_frame, local->op_ret,
+                                  local->op_errno,
+                                  &local->cont.zerofill.prebuf,
+                                  &local->cont.zerofill.postbuf,
+                                  NULL);
+        }
+        return 0;
+}
+
+static int
+afr_zerofill_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t       *local             = NULL;
+        afr_private_t     *priv              = NULL;
+        int                child_index       = (long) cookie;
+        int                call_count        = -1;
+        int                need_unwind       = 0;
+        int                read_child        = 0;
+
+        local = frame->local;
+        priv  = this->private;
+
+        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
+
+        LOCK (&frame->lock);
+        {
+                if (child_index == read_child) {
+                        local->read_child_returned = _gf_true;
+                }
+
+                if (afr_fop_failed (op_ret, op_errno)) {
+                        afr_transaction_fop_failed (frame, this, child_index);
+                }
+
+                if (op_ret != -1) {
+                        if (local->success_count == 0) {
+                                local->op_ret = op_ret;
+                                local->cont.zerofill.prebuf  = *prebuf;
+                                local->cont.zerofill.postbuf = *postbuf;
+                        }
+
+                        if (child_index == read_child) {
+                                local->cont.zerofill.prebuf  = *prebuf;
+                                local->cont.zerofill.postbuf = *postbuf;
+                        }
+
+                        local->success_count++;
+
+                        if ((local->success_count >= priv->wait_count)
+                            && local->read_child_returned) {
+                                need_unwind = 1;
+                        }
+                }
+                local->op_errno = op_errno;
+        }
+        UNLOCK (&frame->lock);
+
+        if (need_unwind) {
+                local->transaction.unwind (frame, this);
+        }
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+        }
+
+        return 0;
+}
+
+static int
+afr_zerofill_wind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t    *local         = NULL;
+        afr_private_t  *priv          = NULL;
+        int             call_count    = -1;
+        int             i             = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_zerofill_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->zerofill,
+                                           local->fd,
+                                           local->cont.zerofill.offset,
+                                           local->cont.zerofill.len,
+                                           NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
+}
+
+static int
+afr_zerofill_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+
+        local = frame->local;
+
+        local->transaction.unwind (frame, this);
+
+        AFR_STACK_DESTROY (frame);
+
+        return 0;
+}
+
+static int
+afr_do_zerofill(call_frame_t *frame, xlator_t *this)
+{
+        call_frame_t  *transaction_frame = NULL;
+        afr_local_t   *local             = NULL;
+        int            op_ret            = -1;
+        int            op_errno          = 0;
+
+        local = frame->local;
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                goto out;
+        }
+
+        transaction_frame->local = local;
+        frame->local = NULL;
+
+        local->op = GF_FOP_ZEROFILL;
+
+        local->transaction.fop    = afr_zerofill_wind;
+        local->transaction.done   = afr_zerofill_done;
+        local->transaction.unwind = afr_zerofill_unwind;
+
+        local->transaction.main_frame = frame;
+
+        local->transaction.start   = local->cont.zerofill.offset;
+        local->transaction.len     = 0;
+
+        op_ret = afr_transaction (transaction_frame, this,
+                                  AFR_DATA_TRANSACTION);
+        if (op_ret < 0) {
+                op_errno = -op_ret;
+                goto out;
+        }
+
+        op_ret = 0;
+out:
+        if (op_ret < 0) {
+                if (transaction_frame) {
+                        AFR_STACK_DESTROY (transaction_frame);
+                }
+                AFR_STACK_UNWIND (zerofill, frame, op_ret, op_errno, NULL,
+                                  NULL, NULL);
+        }
+
+        return 0;
+}
+
+int
+afr_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              size_t len, dict_t *xdata)
+{
+        afr_private_t   *priv               = NULL;
+        afr_local_t     *local              = NULL;
+        call_frame_t    *transaction_frame  = NULL;
+        int              ret                = -1;
+        int              op_errno           = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
+        QUORUM_CHECK(zerofill, out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0) {
+                goto out;
+        }
+        local->cont.zerofill.offset  = offset;
+        local->cont.zerofill.len = len;
+
+        local->fd = fd_ref (fd);
+
+        afr_open_fd_fix (fd, this);
+
+        afr_do_zerofill(frame, this);
+
+        ret = 0;
+out:
+        if (ret < 0) {
+                if (transaction_frame) {
+                        AFR_STACK_DESTROY (transaction_frame);
+                }
+                AFR_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL,
+                                  NULL, NULL);
+        }
+
+        return 0;
+}
+
+/* }}} */
+
+
diff --git a/xlators/cluster/afr/src/afr-inode-write.h b/xlators/cluster/afr/src/afr-inode-write.h
index 729a490d5..8e93ca44a 100644
--- a/xlators/cluster/afr/src/afr-inode-write.h
+++ b/xlators/cluster/afr/src/afr-inode-write.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __INODE_WRITE_H__
@@ -22,59 +13,70 @@
 
 int32_t
 afr_chmod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode);
+	   loc_t *loc, mode_t mode, dict_t *xdata);
 
 int32_t
 afr_chown (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, uid_t uid, gid_t gid);
+	   loc_t *loc, uid_t uid, gid_t gid, dict_t *xdata);
 
 int
 afr_fchown (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, uid_t uid, gid_t gid);
+	    fd_t *fd, uid_t uid, gid_t gid, dict_t *xdata);
 
 int32_t
 afr_fchmod (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, mode_t mode);
+	    fd_t *fd, mode_t mode, dict_t *xdata);
 
 int32_t
 afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 	    struct iovec *vector, int32_t count, off_t offset,
-            uint32_t flags, struct iobref *iobref);
+            uint32_t flags, struct iobref *iobref, dict_t *xdata);
 
 int32_t
 afr_truncate (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, off_t offset);
+	      loc_t *loc, off_t offset, dict_t *xdata);
 
 int32_t
 afr_ftruncate (call_frame_t *frame, xlator_t *this,
-	       fd_t *fd, off_t offset);
+	       fd_t *fd, off_t offset, dict_t *xdata);
 
 int32_t
 afr_utimens (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, struct timespec tv[2]);
+	     loc_t *loc, struct timespec tv[2], dict_t *xdata);
 
 int
 afr_setattr (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, struct iatt *buf, int32_t valid);
+             loc_t *loc, struct iatt *buf, int32_t valid, dict_t *xdata);
 
 int
 afr_fsetattr (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, struct iatt *buf, int32_t valid);
+              fd_t *fd, struct iatt *buf, int32_t valid, dict_t *xdata);
 
 int32_t
 afr_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *dict, int32_t flags);
+              loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata);
 
 int32_t
 afr_fsetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, dict_t *dict, int32_t flags);
+               fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata);
 
 int32_t
 afr_removexattr (call_frame_t *frame, xlator_t *this,
-		 loc_t *loc, const char *name);
+		 loc_t *loc, const char *name, dict_t *xdata);
 
 int32_t
 afr_fremovexattr (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, const char *name);
+                  fd_t *fd, const char *name, dict_t *xdata);
+
+int
+afr_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata);
 
+int
+afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+               off_t offset, size_t len, dict_t *xdata);
+
+int
+afr_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata);
 #endif /* __INODE_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-lk-common.c b/xlators/cluster/afr/src/afr-lk-common.c
index 55dd60e88..060d78f35 100644
--- a/xlators/cluster/afr/src/afr-lk-common.c
+++ b/xlators/cluster/afr/src/afr-lk-common.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "dict.h"
@@ -64,7 +55,36 @@
         } while (0);
 
 int
-afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index);
+afr_entry_lockee_cmp (const void *l1, const void *l2)
+{
+        const afr_entry_lockee_t       *r1 = l1;
+        const afr_entry_lockee_t       *r2 = l2;
+        int                            ret = 0;
+        uuid_t                         gfid1 = {0};
+        uuid_t                         gfid2 = {0};
+
+        loc_gfid ((loc_t*)&r1->loc, gfid1);
+        loc_gfid ((loc_t*)&r2->loc, gfid2);
+        ret = uuid_compare (gfid1, gfid2);
+        /*Entrylks with NULL basename are the 'smallest'*/
+        if (ret == 0) {
+                if (!r1->basename)
+                        return -1;
+                if (!r2->basename)
+                        return 1;
+                ret = strcmp (r1->basename, r2->basename);
+        }
+
+        if (ret <= 0)
+                return -1;
+        else
+                return 1;
+}
+
+int afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index);
+
+static int
+afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this);
 
 static uint64_t afr_lock_number = 1;
 
@@ -131,16 +151,9 @@ internal_lock_count (call_frame_t *frame, xlator_t *this)
         local = frame->local;
         priv  = this->private;
 
-        if (local->fd) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i] && local->fd_open_on[i])
-                                ++call_count;
-                }
-        } else {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i])
-                                ++call_count;
-                }
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i])
+                        ++call_count;
         }
 
         return call_count;
@@ -296,11 +309,10 @@ afr_trace_inodelk_out (call_frame_t *frame, xlator_t *this,
         afr_print_verdict (op_ret, op_errno, verdict);
 
         gf_log (this->name, GF_LOG_INFO,
-                "[%s %s] [%s] Lockee={%s} Number={%llu}",
+                "[%s %s] [%s] lk-owner=%s Lockee={%s} Number={%llu}",
                 lock_call_type_str,
                 lk_op_type == AFR_LOCK_OP ? "LOCK REPLY" : "UNLOCK REPLY",
-                verdict,
-                lockee,
+                verdict, lkowner_utoa (&frame->root->lk_owner), lockee,
                 (unsigned long long) int_lock->lock_number);
 
 }
@@ -339,10 +351,13 @@ static void
 afr_trace_entrylk_in (call_frame_t *frame, xlator_t *this,
                       afr_lock_call_type_t lock_call_type,
                       afr_lock_op_type_t lk_op_type, const char *basename,
-                      int32_t child_index)
+                      int32_t cookie)
 {
         afr_local_t         *local    = NULL;
         afr_internal_lock_t *int_lock = NULL;
+        afr_private_t       *priv     = NULL;
+        int                 child_index = 0;
+        int                 lockee_no = 0;
 
         char lock[256];
         char lockee[256];
@@ -350,28 +365,40 @@ afr_trace_entrylk_in (call_frame_t *frame, xlator_t *this,
 
         local    = frame->local;
         int_lock = &local->internal_lock;
+        priv     = this->private;
+
+        if (!priv->entrylk_trace) {
+                return;
+        }
+        lockee_no = cookie / priv->child_count;
+        child_index = cookie % priv->child_count;
 
         afr_print_entrylk (lock, 256, basename, &frame->root->lk_owner);
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
+        afr_print_lockee (lockee, 256, &int_lock->lockee[lockee_no].loc, local->fd,
+                          child_index);
 
         afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
 
         gf_log (this->name, GF_LOG_INFO,
-                "[%s %s] Lock={%s} Lockee={%s} Number={%llu}",
+                "[%s %s] Lock={%s} Lockee={%s} Number={%llu}, Cookie={%d}",
                 lock_call_type_str,
                 lk_op_type == AFR_LOCK_OP ? "LOCK REQUEST" : "UNLOCK REQUEST",
                 lock, lockee,
-                (unsigned long long) int_lock->lock_number);
+                (unsigned long long) int_lock->lock_number,
+                cookie);
 }
 
 static void
 afr_trace_entrylk_out (call_frame_t *frame, xlator_t *this,
                        afr_lock_call_type_t lock_call_type,
                        afr_lock_op_type_t lk_op_type, const char *basename,
-                       int op_ret, int op_errno, int32_t child_index)
+                       int op_ret, int op_errno, int32_t cookie)
 {
         afr_internal_lock_t *int_lock = NULL;
         afr_local_t         *local    = NULL;
+        afr_private_t       *priv     = NULL;
+        int                 lockee_no = 0;
+        int                 child_index = 0;
 
         char lock[256];
         char lockee[256];
@@ -380,20 +407,30 @@ afr_trace_entrylk_out (call_frame_t *frame, xlator_t *this,
 
         local    = frame->local;
         int_lock = &local->internal_lock;
+        priv     = this->private;
 
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
+        if (!priv->entrylk_trace) {
+                return;
+        }
+        lockee_no = cookie / priv->child_count;
+        child_index = cookie % priv->child_count;
+
+        afr_print_entrylk (lock, 256, basename, &frame->root->lk_owner);
+        afr_print_lockee (lockee, 256, &int_lock->lockee[lockee_no].loc, local->fd,
+                          child_index);
 
         afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
 
         afr_print_verdict (op_ret, op_errno, verdict);
 
         gf_log (this->name, GF_LOG_INFO,
-                "[%s %s] [%s] Lock={%s} Lockee={%s} Number={%llu}",
+                "[%s %s] [%s] Lock={%s} Lockee={%s} Number={%llu} Cookie={%d}",
                 lock_call_type_str,
                 lk_op_type == AFR_LOCK_OP ? "LOCK REPLY" : "UNLOCK REPLY",
                 verdict,
                 lock, lockee,
-                (unsigned long long) int_lock->lock_number);
+                (unsigned long long) int_lock->lock_number,
+                cookie);
 
 }
 
@@ -446,6 +483,47 @@ is_afr_lock_transaction (afr_local_t *local)
         return ret;
 }
 
+int
+afr_init_entry_lockee (afr_entry_lockee_t *lockee, afr_local_t *local,
+                       loc_t *loc, char *basename, int child_count)
+{
+        int     ret     = -1;
+
+        loc_copy (&lockee->loc, loc);
+        lockee->basename        = (basename)? gf_strdup (basename): NULL;
+        if (basename && !lockee->basename)
+                goto out;
+
+        lockee->locked_count    = 0;
+        lockee->locked_nodes    = GF_CALLOC (child_count,
+                                             sizeof (*lockee->locked_nodes),
+                                             gf_afr_mt_afr_node_character);
+
+        if (!lockee->locked_nodes)
+                goto out;
+
+        ret = 0;
+out:
+        return ret;
+
+}
+
+void
+afr_entry_lockee_cleanup (afr_internal_lock_t *int_lock)
+{
+        int     i   = 0;
+
+        for (i = 0; i < int_lock->lockee_count; i++) {
+                loc_wipe (&int_lock->lockee[i].loc);
+                if (int_lock->lockee[i].basename)
+                        GF_FREE (int_lock->lockee[i].basename);
+                if (int_lock->lockee[i].locked_nodes)
+                       GF_FREE (int_lock->lockee[i].locked_nodes);
+        }
+
+        return;
+}
+
 static int
 initialize_entrylk_variables (call_frame_t *frame, xlator_t *this)
 {
@@ -463,8 +541,13 @@ initialize_entrylk_variables (call_frame_t *frame, xlator_t *this)
         int_lock->lock_op_ret        = -1;
         int_lock->lock_op_errno      = 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                int_lock->entry_locked_nodes[i] = 0;
+        for (i = 0; i < AFR_LOCKEE_COUNT_MAX; i++) {
+                if (!int_lock->lockee[i].locked_nodes)
+                        break;
+                int_lock->lockee[i].locked_count = 0;
+                memset (int_lock->lockee[i].locked_nodes, 0,
+                        sizeof (*int_lock->lockee[i].locked_nodes) *
+                        priv->child_count);
         }
 
         return 0;
@@ -476,19 +559,23 @@ initialize_inodelk_variables (call_frame_t *frame, xlator_t *this)
         afr_local_t         *local    = NULL;
         afr_internal_lock_t *int_lock = NULL;
         afr_private_t       *priv     = NULL;
-        int i = 0;
+        afr_inodelk_t       *inodelk  = NULL;
 
         priv     = this->private;
         local    = frame->local;
         int_lock = &local->internal_lock;
 
-        int_lock->inodelk_lock_count = 0;
-        int_lock->lock_op_ret        = -1;
-        int_lock->lock_op_errno      = 0;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
 
-        for (i = 0; i < priv->child_count; i++) {
-                int_lock->inode_locked_nodes[i] = 0;
-        }
+        inodelk->lock_count    = 0;
+        int_lock->lk_attempted_count = 0;
+        int_lock->lock_op_ret   = -1;
+        int_lock->lock_op_errno = 0;
+
+        memset (inodelk->locked_nodes, 0,
+                sizeof (*inodelk->locked_nodes) * priv->child_count);
+        memset (int_lock->locked_nodes, 0,
+                sizeof (*int_lock->locked_nodes) * priv->child_count);
 
         return 0;
 }
@@ -510,6 +597,18 @@ lower_path (loc_t *l1, const char *b1, loc_t *l2, const char *b2)
 }
 
 int
+afr_lockee_locked_nodes_count (afr_internal_lock_t *int_lock)
+{
+        int call_count  = 0;
+        int i           = 0;
+
+        for (i = 0; i < int_lock->lockee_count; i++)
+                call_count += int_lock->lockee[i].locked_count;
+
+        return call_count;
+}
+
+int
 afr_locked_nodes_count (unsigned char *locked_nodes, int child_count)
 
 {
@@ -527,7 +626,7 @@ afr_locked_nodes_count (unsigned char *locked_nodes, int child_count)
 /* FIXME: What if UNLOCK fails */
 static int32_t
 afr_unlock_common_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t         *local    = NULL;
         afr_internal_lock_t *int_lock = NULL;
@@ -553,11 +652,13 @@ afr_unlock_common_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 static int32_t
 afr_unlock_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t         *local = NULL;
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk = NULL;
         int32_t             child_index = (long)cookie;
+        afr_private_t       *priv = NULL;
 
         local = frame->local;
         int_lock = &local->internal_lock;
@@ -566,18 +667,22 @@ afr_unlock_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                AFR_UNLOCK_OP, NULL, op_ret,
                                op_errno, child_index);
 
+        priv = this->private;
+
         if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
-                gf_log (this->name, GF_LOG_INFO, "%s: unlock failed on %d "
-                        "unlock by %s", local->loc.path, child_index,
+                gf_log (this->name, GF_LOG_INFO, "%s: unlock failed on subvolume %s "
+                        "with lock owner %s", local->loc.path,
+                        priv->children[child_index]->name,
                         lkowner_utoa (&frame->root->lk_owner));
         }
 
 
-        int_lock->inode_locked_nodes[child_index] &= LOCKED_NO;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+        inodelk->locked_nodes[child_index] &= LOCKED_NO;
         if (local->transaction.eager_lock)
                 local->transaction.eager_lock[child_index] = 0;
 
-        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno);
+        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno, xdata);
 
         return 0;
 
@@ -587,6 +692,7 @@ static int
 afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
         afr_private_t       *priv     = NULL;
         struct gf_flock flock = {0,};
@@ -596,19 +702,20 @@ afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
         int i = 0;
         int piggyback = 0;
         afr_fd_ctx_t        *fd_ctx      = NULL;
-        gf_boolean_t    fd_lock_owner = _gf_false;
 
 
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
 
-        flock.l_start = int_lock->lk_flock.l_start;
-        flock.l_len   = int_lock->lk_flock.l_len;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+
+        flock.l_start = inodelk->flock.l_start;
+        flock.l_len   = inodelk->flock.l_len;
         flock.l_type  = F_UNLCK;
 
         full_flock.l_type = F_UNLCK;
-        call_count = afr_locked_nodes_count (int_lock->inode_locked_nodes,
+        call_count = afr_locked_nodes_count (inodelk->locked_nodes,
                                              priv->child_count);
 
         int_lock->lk_call_count = call_count;
@@ -624,18 +731,12 @@ afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
                 fd_ctx = afr_fd_ctx_get (local->fd, this);
 
         for (i = 0; i < priv->child_count; i++) {
-                if ((int_lock->inode_locked_nodes[i] & LOCKED_YES)
-                    != LOCKED_YES)
+                if ((inodelk->locked_nodes[i] & LOCKED_YES) != LOCKED_YES)
                         continue;
 
                 if (local->fd) {
                         flock_use = &flock;
                         if (!local->transaction.eager_lock[i]) {
-                                if (fd_lock_owner) {
-                                        afr_set_lk_owner (frame, this,
-                                                          frame->root);
-                                        fd_lock_owner = _gf_false;
-                                }
                                 goto wind;
                         }
 
@@ -654,16 +755,12 @@ afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
 
                         if (piggyback) {
                                 afr_unlock_inodelk_cbk (frame, (void *) (long) i,
-                                                        this, 1, 0);
+                                                        this, 1, 0, NULL);
                                 if (!--call_count)
                                         break;
                                 continue;
                         }
 
-                        if (!fd_lock_owner) {
-                                afr_set_lk_owner (frame, this, local->fd);
-                                fd_lock_owner = _gf_true;
-                        }
                         flock_use = &full_flock;
                 wind:
                         AFR_TRACE_INODELK_IN (frame, this,
@@ -675,8 +772,8 @@ afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
                                            (void *) (long)i,
                                            priv->children[i],
                                            priv->children[i]->fops->finodelk,
-                                           this->name, local->fd,
-                                           F_SETLK, flock_use);
+                                           int_lock->domain, local->fd,
+                                           F_SETLK, flock_use, NULL);
 
                         if (!--call_count)
                                 break;
@@ -690,8 +787,8 @@ afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
                                            (void *) (long)i,
                                            priv->children[i],
                                            priv->children[i]->fops->inodelk,
-                                           this->name, &local->loc,
-                                           F_SETLK, &flock);
+                                           int_lock->domain, &local->loc,
+                                           F_SETLK, &flock, NULL);
 
                         if (!--call_count)
                                 break;
@@ -703,16 +800,25 @@ out:
 
 static int32_t
 afr_unlock_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t         *local = NULL;
-        int32_t             child_index = (long)cookie;
+        afr_private_t       *priv  = NULL;
+        afr_internal_lock_t *int_lock = NULL;
+        int32_t             child_index = 0;
+        int                 lockee_no   = 0;
+
+        priv = this->private;
+        lockee_no = (int)((long) cookie) / priv->child_count;
+        child_index = (int) ((long) cookie) % priv->child_count;
 
         local = frame->local;
+        int_lock = &local->internal_lock;
 
         AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
-                               AFR_UNLOCK_OP, NULL, op_ret,
-                               op_errno, child_index);
+                               AFR_UNLOCK_OP,
+                               int_lock->lockee[lockee_no].basename, op_ret,
+                               op_errno, (int) ((long)cookie));
 
         if (op_ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR,
@@ -720,7 +826,8 @@ afr_unlock_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->loc.path, child_index, strerror (op_errno));
         }
 
-        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno);
+        int_lock->lockee[lockee_no].locked_nodes[child_index] &= LOCKED_NO;
+        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno, NULL);
 
         return 0;
 }
@@ -728,24 +835,22 @@ afr_unlock_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 static int
 afr_unlock_entrylk (call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        const char          *basename = NULL;
-        loc_t               *loc      = NULL;
-        int call_count = 0;
-        int i = -1;
+        afr_internal_lock_t     *int_lock       = NULL;
+        afr_local_t             *local          = NULL;
+        afr_private_t           *priv           = NULL;
+        int                     call_count      = 0;
+        int                     index           = 0;
+        int                     lockee_no       = 0;
+        int                     copies          = 0;
+        int                     i               = -1;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
+        copies   = priv->child_count;
 
-        basename = int_lock->lk_basename;
-        if (int_lock->lk_loc)
-                loc = int_lock->lk_loc;
+        call_count = afr_lockee_locked_nodes_count (int_lock);
 
-        call_count = afr_locked_nodes_count (int_lock->entry_locked_nodes,
-                                             priv->child_count);
         int_lock->lk_call_count = call_count;
 
         if (!call_count){
@@ -755,19 +860,23 @@ afr_unlock_entrylk (call_frame_t *frame, xlator_t *this)
                 goto out;
         }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->entry_locked_nodes[i] & LOCKED_YES) {
-                        AFR_TRACE_ENTRYLK_IN (frame, this,
-                                              AFR_ENTRYLK_NB_TRANSACTION,
-                                              AFR_UNLOCK_OP, basename, i);
+        for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) {
+                lockee_no = i / copies;
+                index     = i % copies;
+                if (int_lock->lockee[lockee_no].locked_nodes[index] & LOCKED_YES) {
+                        AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
+                                              AFR_UNLOCK_OP,
+                                              int_lock->lockee[lockee_no].basename,
+                                              i);
 
                         STACK_WIND_COOKIE (frame, afr_unlock_entrylk_cbk,
                                            (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->entrylk,
-                                           this->name,
-                                           loc, basename,
-                                           ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
+                                           priv->children[index],
+                                           priv->children[index]->fops->entrylk,
+                                           int_lock->domain,
+                                           &int_lock->lockee[lockee_no].loc,
+                                           int_lock->lockee[lockee_no].basename,
+                                           ENTRYLK_UNLOCK, ENTRYLK_WRLCK, NULL);
 
                         if (!--call_count)
                                 break;
@@ -781,15 +890,22 @@ out:
 
 static int32_t
 afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno)
+              int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        int child_index = (long) cookie;
+        afr_internal_lock_t     *int_lock       = NULL;
+        afr_local_t             *local          = NULL;
+        afr_private_t           *priv           = NULL;
+        int                     cky             = (long) cookie;
+        int                     child_index     = 0;
+        int                     lockee_no       = 0;
 
+        priv     = this->private;
         local    = frame->local;
         int_lock = &local->internal_lock;
 
+        child_index = ((int)cky) % priv->child_count;
+        lockee_no   = ((int)cky) / priv->child_count;
+
         LOCK (&frame->lock);
         {
                 if (op_ret == -1) {
@@ -805,6 +921,8 @@ afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->op_errno              = op_errno;
                         int_lock->lock_op_errno      = op_errno;
                 }
+
+		int_lock->lk_attempted_count++;
         }
         UNLOCK (&frame->lock);
 
@@ -813,10 +931,17 @@ afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 afr_unlock (frame, this);
         } else {
                 if (op_ret == 0) {
-                        int_lock->locked_nodes[child_index] |= LOCKED_YES;
-                        int_lock->lock_count++;
+                        if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
+                            local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
+                                int_lock->lockee[lockee_no].locked_nodes[child_index] |= LOCKED_YES;
+                                int_lock->lockee[lockee_no].locked_count++;
+                                int_lock->entrylk_lock_count++;
+                        } else {
+                                int_lock->locked_nodes[child_index] |= LOCKED_YES;
+                                int_lock->lock_count++;
+                        }
                 }
-                afr_lock_blocking (frame, this, child_index + 1);
+                afr_lock_blocking (frame, this, cky + 1);
         }
 
         return 0;
@@ -824,98 +949,26 @@ afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 static int32_t
 afr_blocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_TRANSACTION,
                                AFR_LOCK_OP, NULL, op_ret,
                                op_errno, (long) cookie);
 
-        afr_lock_cbk (frame, cookie, this, op_ret, op_errno);
+        afr_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
         return 0;
 
 }
 
 static int32_t
-afr_lock_lower_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_private_t       *priv        = NULL;
-        afr_local_t         *local       = NULL;
-        loc_t               *lower       = NULL;
-        loc_t               *higher      = NULL;
-        const char          *higher_name = NULL;
-        int child_index = (long) cookie;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        if (op_errno == ENOSYS) {
-                                /* return ENOTSUP */
-
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "subvolume does not support locking. "
-                                        "please load features/locks xlator on server");
-
-                                local->op_ret   = op_ret;
-                        }
-
-                        local->op_errno = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (op_ret != 0) {
-                afr_unlock (frame, this);
-                goto out;
-        } else {
-                int_lock->lower_locked_nodes[child_index] |= LOCKED_LOWER;
-                int_lock->lock_count++;
-        }
-
-        /* The lower path has been locked. Now lock the higher path */
-
-        lower = lower_path (&local->transaction.parent_loc,
-                            local->transaction.basename,
-                            &local->transaction.new_parent_loc,
-                            local->transaction.new_basename);
-
-        higher = (lower == &local->transaction.parent_loc ?
-                  &local->transaction.new_parent_loc :
-                  &local->transaction.parent_loc);
-
-        higher_name = (higher == &local->transaction.parent_loc ?
-                       local->transaction.basename :
-                       local->transaction.new_basename);
-
-        AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_TRANSACTION,
-                              AFR_LOCK_OP, higher_name, child_index);
-
-
-        STACK_WIND_COOKIE (frame, afr_lock_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->entrylk,
-                           this->name, higher, higher_name,
-                           ENTRYLK_LOCK, ENTRYLK_WRLCK);
-
-out:
-        return 0;
-}
-
-static int32_t
 afr_blocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
                                AFR_LOCK_OP, NULL, op_ret,
                                op_errno, (long)cookie);
 
-        afr_lock_cbk (frame, cookie, this, op_ret, op_errno);
+        afr_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -923,6 +976,7 @@ static int
 afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
         afr_private_t       *priv     = NULL;
 
@@ -933,18 +987,16 @@ afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this)
         switch (local->transaction.type) {
         case AFR_DATA_TRANSACTION:
         case AFR_METADATA_TRANSACTION:
-                memcpy (int_lock->inode_locked_nodes,
-                        int_lock->locked_nodes,
-                        priv->child_count);
-                int_lock->inodelk_lock_count = int_lock->lock_count;
+                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+                memcpy (inodelk->locked_nodes, int_lock->locked_nodes,
+                        sizeof (*inodelk->locked_nodes) * priv->child_count);
+                inodelk->lock_count = int_lock->lock_count;
                 break;
 
         case AFR_ENTRY_RENAME_TRANSACTION:
         case AFR_ENTRY_TRANSACTION:
-                memcpy (int_lock->entry_locked_nodes,
-                        int_lock->locked_nodes,
-                        priv->child_count);
-                int_lock->entrylk_lock_count = int_lock->lock_count;
+                /*entrylk_count is being used in both non-blocking and blocking
+                 * modes */
                 break;
         }
 
@@ -952,25 +1004,67 @@ afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this)
 
 }
 
+static inline gf_boolean_t
+afr_is_entrylk (afr_internal_lock_t *int_lock,
+                afr_transaction_type trans_type)
+{
+        gf_boolean_t is_entrylk = _gf_false;
+
+        if ((int_lock->transaction_lk_type == AFR_SELFHEAL_LK) &&
+            int_lock->selfheal_lk_type == AFR_ENTRY_SELF_HEAL_LK) {
+
+                is_entrylk = _gf_true;
+
+        } else if ((int_lock->transaction_lk_type == AFR_TRANSACTION_LK) &&
+                 (trans_type == AFR_ENTRY_TRANSACTION ||
+                  trans_type == AFR_ENTRY_RENAME_TRANSACTION)) {
+
+                is_entrylk = _gf_true;
+
+        } else {
+                is_entrylk = _gf_false;
+        }
+
+        return is_entrylk;
+}
+
+static gf_boolean_t
+_is_lock_wind_needed (afr_local_t *local, int child_index)
+{
+        if (!local->child_up[child_index])
+                return _gf_false;
+
+        return _gf_true;
+}
+
 int
-afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
+afr_lock_blocking (call_frame_t *frame, xlator_t *this, int cookie)
 {
         afr_internal_lock_t *int_lock    = NULL;
+        afr_inodelk_t       *inodelk     = NULL;
         afr_local_t         *local       = NULL;
         afr_private_t       *priv        = NULL;
-        loc_t               *lower       = NULL;
-        const char          *lower_name  = NULL;
         struct gf_flock flock = {0,};
         uint64_t ctx = 0;
         int ret = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        flock.l_start = int_lock->lk_flock.l_start;
-        flock.l_len   = int_lock->lk_flock.l_len;
-        flock.l_type  = int_lock->lk_flock.l_type;
+        int child_index = 0;
+        int lockee_no   = 0;
+        gf_boolean_t is_entrylk = _gf_false;
+
+        local         = frame->local;
+        int_lock      = &local->internal_lock;
+        priv          = this->private;
+        child_index   = cookie % priv->child_count;
+        lockee_no     = cookie / priv->child_count;
+        is_entrylk    = afr_is_entrylk (int_lock, local->transaction.type);
+
+
+        if (!is_entrylk) {
+                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+                flock.l_start = inodelk->flock.l_start;
+                flock.l_len   = inodelk->flock.l_len;
+                flock.l_type  = inodelk->flock.l_type;
+        }
 
         if (local->fd) {
                 ret = fd_ctx_get (local->fd, this, &ctx);
@@ -989,42 +1083,26 @@ afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
 
                         return 0;
                 }
-
-                /* skip over children that or down
-                   or don't have the fd open */
-
-                while ((child_index < priv->child_count)
-                       && (!local->child_up[child_index] ||
-                          !local->fd_open_on[child_index]))
-
-                        child_index++;
-        } else {
-                /* skip over children that are down */
-                while ((child_index < priv->child_count)
-                       && !local->child_up[child_index])
-                        child_index++;
         }
 
-        if ((child_index == priv->child_count) &&
-            int_lock->lock_count == 0) {
-
-                gf_log (this->name, GF_LOG_INFO,
-                        "unable to lock on even one child");
-
-                local->op_ret           = -1;
-                int_lock->lock_op_ret   = -1;
+        if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
+                if ((is_entrylk && int_lock->entrylk_lock_count == 0) ||
+                    (!is_entrylk && int_lock->lock_count == 0)) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "unable to lock on even one child");
 
-                afr_copy_locked_nodes (frame, this);
+                        local->op_ret           = -1;
+                        int_lock->lock_op_ret   = -1;
 
-                afr_unlock(frame, this);
+                        afr_copy_locked_nodes (frame, this);
 
-                return 0;
+                        afr_unlock(frame, this);
 
+                        return 0;
+                }
         }
 
-        if ((child_index == priv->child_count)
-            || (int_lock->lock_count == int_lock->lk_expected_count)) {
-
+        if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
                 /* we're done locking */
 
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -1037,6 +1115,11 @@ afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
                 return 0;
         }
 
+        if (!_is_lock_wind_needed (local, child_index)) {
+                afr_lock_blocking (frame, this, cookie + 1);
+                return 0;
+        }
+
         switch (local->transaction.type) {
         case AFR_DATA_TRANSACTION:
         case AFR_METADATA_TRANSACTION:
@@ -1051,8 +1134,8 @@ afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
                                            (void *) (long) child_index,
                                            priv->children[child_index],
                                            priv->children[child_index]->fops->finodelk,
-                                           this->name, local->fd,
-                                           F_SETLKW, &flock);
+                                           int_lock->domain, local->fd,
+                                           F_SETLKW, &flock, NULL);
 
                 } else {
                         AFR_TRACE_INODELK_IN (frame, this,
@@ -1064,51 +1147,30 @@ afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
                                            (void *) (long) child_index,
                                            priv->children[child_index],
                                            priv->children[child_index]->fops->inodelk,
-                                           this->name, &local->loc,
-                                           F_SETLKW, &flock);
+                                           int_lock->domain, &local->loc,
+                                           F_SETLKW, &flock, NULL);
                 }
 
                 break;
 
         case AFR_ENTRY_RENAME_TRANSACTION:
-        {
-                lower = lower_path (&local->transaction.parent_loc,
-                                    local->transaction.basename,
-                                    &local->transaction.new_parent_loc,
-                                    local->transaction.new_basename);
-
-                lower_name = (lower == &local->transaction.parent_loc ?
-                              local->transaction.basename :
-                              local->transaction.new_basename);
-
-                AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_TRANSACTION,
-                                      AFR_LOCK_OP, lower_name, child_index);
-
-
-                STACK_WIND_COOKIE (frame, afr_lock_lower_cbk,
-                                   (void *) (long) child_index,
-                                   priv->children[child_index],
-                                   priv->children[child_index]->fops->entrylk,
-                                   this->name, lower, lower_name,
-                                   ENTRYLK_LOCK, ENTRYLK_WRLCK);
-
-                break;
-        }
-
         case AFR_ENTRY_TRANSACTION:
+                /*Accounting for child_index increments on 'down'
+                 *and 'fd-less' children */
+
                 if (local->fd) {
-                        AFR_TRACE_ENTRYLK_IN (frame, this,
-                                              AFR_ENTRYLK_TRANSACTION,
-                                              AFR_LOCK_OP, local->transaction.basename,
-                                              child_index);
+                        AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_TRANSACTION,
+                                              AFR_LOCK_OP,
+                                              int_lock->lockee[lockee_no].basename,
+                                              cookie);
 
                         STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
-                                           (void *) (long) child_index,
+                                           (void *) (long) cookie,
                                            priv->children[child_index],
                                            priv->children[child_index]->fops->fentrylk,
-                                           this->name, local->fd,
-                                           local->transaction.basename,
-                                           ENTRYLK_LOCK, ENTRYLK_WRLCK);
+                                           int_lock->domain, local->fd,
+                                           int_lock->lockee[lockee_no].basename,
+                                           ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
                 } else {
                         AFR_TRACE_ENTRYLK_IN (frame, this,
                                               AFR_ENTRYLK_TRANSACTION,
@@ -1116,13 +1178,13 @@ afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
                                               child_index);
 
                         STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
-                                           (void *) (long) child_index,
+                                           (void *) (long) cookie,
                                            priv->children[child_index],
                                            priv->children[child_index]->fops->entrylk,
-                                           this->name,
-                                           &local->transaction.parent_loc,
-                                           local->transaction.basename,
-                                           ENTRYLK_LOCK, ENTRYLK_WRLCK);
+                                           int_lock->domain,
+                                           &int_lock->lockee[lockee_no].loc,
+                                           int_lock->lockee[lockee_no].basename,
+                                           ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
                 }
 
                 break;
@@ -1150,11 +1212,12 @@ afr_blocking_lock (call_frame_t *frame, xlator_t *this)
                 break;
 
         case AFR_ENTRY_RENAME_TRANSACTION:
+        case AFR_ENTRY_TRANSACTION:
                 up_count = afr_up_children_count (local->child_up,
                                                   priv->child_count);
-                int_lock->lk_expected_count = 2 * up_count;
-                //fallthrough
-        case AFR_ENTRY_TRANSACTION:
+                int_lock->lk_call_count = int_lock->lk_expected_count
+                                        = (int_lock->lockee_count *
+                                           up_count);
                 initialize_entrylk_variables (frame, this);
                 break;
         }
@@ -1166,42 +1229,55 @@ afr_blocking_lock (call_frame_t *frame, xlator_t *this)
 
 static int32_t
 afr_nonblocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno)
+                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_internal_lock_t *int_lock = NULL;
         afr_local_t         *local    = NULL;
-        int                 call_count  = 0;
-        int                 child_index = (long) cookie;
+        int call_count          = 0;
+        int child_index         = (long) cookie;
+        int copies              = 0;
+        int index               = 0;
+        int lockee_no           = 0;
+        afr_private_t       *priv = NULL;
+
+        priv = this->private;
+
+        copies = priv->child_count;
+        index = child_index % copies;
+        lockee_no = child_index / copies;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
 
         AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
-                               AFR_LOCK_OP, NULL, op_ret,
+                               AFR_LOCK_OP,
+                               int_lock->lockee[lockee_no].basename, op_ret,
                                op_errno, (long) cookie);
 
-        LOCK (&frame->lock);
-        {
-                call_count = --int_lock->lk_call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (op_ret < 0 ) {
-                if (op_errno == ENOSYS) {
+	LOCK (&frame->lock);
+	{
+		if (op_ret < 0 ) {
+			if (op_errno == ENOSYS) {
                         /* return ENOTSUP */
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "subvolume does not support locking. "
-                                "please load features/locks xlator on server");
-                        local->op_ret         = op_ret;
-                        int_lock->lock_op_ret = op_ret;
+				gf_log (this->name, GF_LOG_ERROR,
+					"subvolume does not support locking. "
+					"please load features/locks xlator on server");
+				local->op_ret         = op_ret;
+				int_lock->lock_op_ret = op_ret;
+
+				int_lock->lock_op_errno      = op_errno;
+				local->op_errno              = op_errno;
+			}
+		} else if (op_ret == 0) {
+			int_lock->lockee[lockee_no].locked_nodes[index] |= \
+				LOCKED_YES;
+			int_lock->lockee[lockee_no].locked_count++;
+			int_lock->entrylk_lock_count++;
+		}
 
-                        int_lock->lock_op_errno      = op_errno;
-                        local->op_errno              = op_errno;
-                }
-        } else if (op_ret == 0) {
-                int_lock->entry_locked_nodes[child_index] |= LOCKED_YES;
-                int_lock->entrylk_lock_count++;
+                call_count = --int_lock->lk_call_count;
         }
+        UNLOCK (&frame->lock);
 
         if (call_count == 0) {
                 gf_log (this->name, GF_LOG_TRACE,
@@ -1228,42 +1304,26 @@ afr_nonblocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
-void
-afr_mark_fd_open_on (afr_local_t *local, afr_fd_ctx_t *fd_ctx,
-                       size_t child_count)
-{
-        int             i = 0;
-
-        GF_ASSERT (local->fd_open_on);
-
-        memset (local->fd_open_on, 0, sizeof (*local->fd_open_on)*child_count);
-        for (i = 0; i < child_count; i++)
-                if (fd_ctx->opened_on[i] == AFR_FD_OPENED)
-                        local->fd_open_on[i] = 1;
-}
-
 int
 afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_fd_ctx_t        *fd_ctx   = NULL;
-        const char          *basename = NULL;
-        loc_t               *loc      = NULL;
-        int32_t call_count = 0;
+        afr_internal_lock_t *int_lock   = NULL;
+        afr_local_t         *local      = NULL;
+        afr_private_t       *priv       = NULL;
+        afr_fd_ctx_t        *fd_ctx     = NULL;
+        int                 copies      = 0;
+        int                 index       = 0;
+        int                 lockee_no   = 0;
+        int32_t             call_count  = 0;
         int i = 0;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
 
+        copies = priv->child_count;
         initialize_entrylk_variables (frame, this);
 
-        basename = int_lock->lk_basename;
-        if (int_lock->lk_loc)
-                loc = int_lock->lk_loc;
-
         if (local->fd) {
                 fd_ctx = afr_fd_ctx_get (local->fd, this);
                 if (!fd_ctx) {
@@ -1276,11 +1336,11 @@ afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this)
                         local->op_errno         = EINVAL;
                         int_lock->lock_op_errno = EINVAL;
 
+			afr_unlock (frame, this);
                         return -1;
                 }
 
-                afr_mark_fd_open_on (local, fd_ctx, priv->child_count);
-                call_count = internal_lock_count (frame, this);
+                call_count = int_lock->lockee_count * internal_lock_count (frame, this);
                 int_lock->lk_call_count = call_count;
                 int_lock->lk_expected_count = call_count;
 
@@ -1293,44 +1353,52 @@ afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this)
 
                 /* Send non-blocking entrylk calls only on up children
                    and where the fd has been opened */
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i] && local->fd_open_on[i]) {
-                                AFR_TRACE_ENTRYLK_IN (frame, this,
-                                                      AFR_ENTRYLK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP, basename, i);
+                for (i = 0; i < int_lock->lockee_count*priv->child_count; i++) {
+                        index = i%copies;
+                        lockee_no = i/copies;
+                        if (local->child_up[index]) {
+                                AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
+                                                      AFR_LOCK_OP,
+                                                      int_lock->lockee[lockee_no].basename,
+                                                      i);
 
                                 STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
                                                    (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fentrylk,
+                                                   priv->children[index],
+                                                   priv->children[index]->fops->fentrylk,
                                                    this->name, local->fd,
-                                                   basename,
-                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK);
+                                                   int_lock->lockee[lockee_no].basename,
+                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
+                                                   NULL);
+                                if (!--call_count)
+                                        break;
                         }
                 }
         } else {
-                GF_ASSERT (loc);
-
-                call_count = internal_lock_count (frame, this);
+                call_count = int_lock->lockee_count * internal_lock_count (frame, this);
                 int_lock->lk_call_count = call_count;
                 int_lock->lk_expected_count = call_count;
 
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i]) {
-                                AFR_TRACE_ENTRYLK_IN (frame, this,
-                                                      AFR_ENTRYLK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP, basename, i);
+                for (i = 0; i < int_lock->lockee_count*priv->child_count; i++) {
+                        index = i%copies;
+                        lockee_no = i/copies;
+                        if (local->child_up[index]) {
+                                AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
+                                                      AFR_LOCK_OP,
+                                                      int_lock->lockee[lockee_no].basename,
+                                                      i);
 
                                 STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
                                                    (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->entrylk,
-                                                   this->name, loc, basename,
-                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK);
+                                                   priv->children[index],
+                                                   priv->children[index]->fops->entrylk,
+                                                   this->name, &int_lock->lockee[lockee_no].loc,
+                                                   int_lock->lockee[lockee_no].basename,
+                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
+                                                   NULL);
 
                                 if (!--call_count)
                                         break;
-
                         }
                 }
         }
@@ -1340,9 +1408,10 @@ out:
 
 int32_t
 afr_nonblocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno)
+                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
         int call_count  = 0;
         int child_index = (long) cookie;
@@ -1351,58 +1420,57 @@ afr_nonblocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local    = frame->local;
         int_lock = &local->internal_lock;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
 
         AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_NB_TRANSACTION,
                                AFR_LOCK_OP, NULL, op_ret,
                                op_errno, (long) cookie);
 
-        if (op_ret < 0) {
-                if (op_errno == ENOSYS) {
-                        /* return ENOTSUP */
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "subvolume does not support locking. "
-                                "please load features/locks xlator on server");
-                        local->op_ret                = op_ret;
-                        int_lock->lock_op_ret        = op_ret;
-                        int_lock->lock_op_errno      = op_errno;
-                        local->op_errno              = op_errno;
-                }
-                if (local->transaction.eager_lock)
-                        local->transaction.eager_lock[child_index] = 0;
-        } else {
-                int_lock->inode_locked_nodes[child_index]
-                        |= LOCKED_YES;
-                int_lock->inodelk_lock_count++;
-
-                if (local->transaction.eager_lock &&
-                    local->transaction.eager_lock[child_index] && local->fd) {
-                        fd_ctx = afr_fd_ctx_get (local->fd, this);
-                        /* piggybacked */
-
-                        if (op_ret == 1) {
-                                /* piggybacked */
-                        } else if (op_ret == 0) {
-                                /* lock acquired from server */
-                                LOCK (&local->fd->lock);
-                                {
-                                        fd_ctx->lock_acquired[child_index]++;
-                                }
-                                UNLOCK (&local->fd->lock);
-                        }
-                }
-        }
+	if (local->fd)
+		fd_ctx = afr_fd_ctx_get (local->fd, this);
 
         LOCK (&frame->lock);
         {
+		if (op_ret < 0) {
+			if (op_errno == ENOSYS) {
+				/* return ENOTSUP */
+				gf_log (this->name, GF_LOG_ERROR,
+					"subvolume does not support locking. "
+					"please load features/locks xlator on "
+					"server");
+				local->op_ret                = op_ret;
+				int_lock->lock_op_ret        = op_ret;
+				int_lock->lock_op_errno      = op_errno;
+				local->op_errno              = op_errno;
+			}
+			if (local->transaction.eager_lock)
+				local->transaction.eager_lock[child_index] = 0;
+		} else {
+			inodelk->locked_nodes[child_index] |= LOCKED_YES;
+			inodelk->lock_count++;
+
+			if (local->transaction.eager_lock &&
+			    local->transaction.eager_lock[child_index] &&
+			    local->fd) {
+				/* piggybacked */
+				if (op_ret == 1) {
+					/* piggybacked */
+				} else if (op_ret == 0) {
+					/* lock acquired from server */
+                                        fd_ctx->lock_acquired[child_index]++;
+				}
+			}
+		}
+
                 call_count = --int_lock->lk_call_count;
         }
         UNLOCK (&frame->lock);
+
         if (call_count == 0) {
                 gf_log (this->name, GF_LOG_TRACE,
                         "Last inode locking reply received");
                 /* all locks successful. Proceed to call FOP */
-                if (int_lock->inodelk_lock_count ==
-                                int_lock->lk_expected_count) {
+                if (inodelk->lock_count == int_lock->lk_expected_count) {
                         gf_log (this->name, GF_LOG_TRACE,
                                 "All servers locked. Calling the cbk");
                         int_lock->lock_op_ret = 0;
@@ -1426,6 +1494,7 @@ int
 afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
         afr_private_t       *priv     = NULL;
         afr_fd_ctx_t        *fd_ctx   = NULL;
@@ -1436,17 +1505,18 @@ afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
         struct              gf_flock full_flock = {0,};
         struct              gf_flock *flock_use = NULL;
         int                 piggyback = 0;
-        gf_boolean_t        fd_lock_owner = _gf_false;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
 
-        flock.l_start = int_lock->lk_flock.l_start;
-        flock.l_len   = int_lock->lk_flock.l_len;
-        flock.l_type  = int_lock->lk_flock.l_type;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
 
-        full_flock.l_type = int_lock->lk_flock.l_type;
+        flock.l_start = inodelk->flock.l_start;
+        flock.l_len   = inodelk->flock.l_len;
+        flock.l_type  = inodelk->flock.l_type;
+
+        full_flock.l_type = inodelk->flock.l_type;
 
         initialize_inodelk_variables (frame, this);
 
@@ -1462,11 +1532,11 @@ afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
                         local->op_errno         = EINVAL;
                         int_lock->lock_op_errno = EINVAL;
 
+			afr_unlock (frame, this);
                         ret = -1;
                         goto out;
                 }
 
-                afr_mark_fd_open_on (local, fd_ctx, priv->child_count);
                 call_count = internal_lock_count (frame, this);
                 int_lock->lk_call_count = call_count;
                 int_lock->lk_expected_count = call_count;
@@ -1481,22 +1551,19 @@ afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
                 /* Send non-blocking inodelk calls only on up children
                    and where the fd has been opened */
                 for (i = 0; i < priv->child_count; i++) {
-                        if (!local->child_up[i] || !local->fd_open_on[i])
+                        if (!local->child_up[i])
                                 continue;
 
                         flock_use = &flock;
-                        if (!priv->eager_lock) {
-                                if (fd_lock_owner) {
-                                        afr_set_lk_owner (frame, this,
-                                                          frame->root);
-                                        fd_lock_owner = _gf_false;
-                                }
+                        if (!local->transaction.eager_lock_on) {
                                 goto wind;
                         }
 
                         piggyback = 0;
                         local->transaction.eager_lock[i] = 1;
 
+			afr_set_delayed_post_op (frame, this);
+
                         LOCK (&local->fd->lock);
                         {
                                 if (fd_ctx->lock_acquired[i]) {
@@ -1509,16 +1576,12 @@ afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
                         if (piggyback) {
                                 /* (op_ret == 1) => indicate piggybacked lock */
                                 afr_nonblocking_inodelk_cbk (frame, (void *) (long) i,
-                                                             this, 1, 0);
+                                                             this, 1, 0, NULL);
                                 if (!--call_count)
                                         break;
                                 continue;
                         }
                         flock_use = &full_flock;
-                        if (!fd_lock_owner) {
-                                afr_set_lk_owner (frame, this, local->fd);
-                                fd_lock_owner = _gf_true;
-                        }
                 wind:
                         AFR_TRACE_INODELK_IN (frame, this,
                                               AFR_INODELK_NB_TRANSACTION,
@@ -1528,8 +1591,8 @@ afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->finodelk,
-                                           this->name, local->fd,
-                                           F_SETLK, flock_use);
+                                           int_lock->domain, local->fd,
+                                           F_SETLK, flock_use, NULL);
 
                         if (!--call_count)
                                 break;
@@ -1550,8 +1613,8 @@ afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->inodelk,
-                                           this->name, &local->loc,
-                                           F_SETLK, &flock);
+                                           int_lock->domain, &local->loc,
+                                           F_SETLK, &flock, NULL);
 
                         if (!--call_count)
                                 break;
@@ -1561,201 +1624,6 @@ out:
         return ret;
 }
 
-static int
-__is_lower_locked (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_local_t         *local    = NULL;
-        int count = 0;
-        int i     = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->lower_locked_nodes[i] & LOCKED_LOWER)
-                        count++;
-        }
-
-        return count;
-
-}
-
-static int
-__is_higher_locked (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_local_t         *local    = NULL;
-        int count = 0;
-        int i = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->locked_nodes[i] & LOCKED_YES)
-                        count++;
-        }
-
-        return count;
-
-}
-
-static int
-afr_unlock_lower_entrylk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        const char          *basename = NULL;
-        loc_t               *loc      = NULL;
-        int call_count = 0;
-        int i = -1;
-
-        local = frame->local;
-        int_lock = &local->internal_lock;
-        priv  = this->private;
-
-        basename = int_lock->lk_basename;
-        if (int_lock->lk_loc)
-                loc = int_lock->lk_loc;
-
-        call_count = __is_lower_locked (frame, this);
-        int_lock->lk_call_count = call_count;
-
-        if (!call_count){
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No internal locks unlocked");
-                int_lock->lock_cbk (frame, this);
-                goto out;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->lower_locked_nodes[i] & LOCKED_LOWER) {
-                        AFR_TRACE_ENTRYLK_IN (frame, this,
-                                              AFR_ENTRYLK_NB_TRANSACTION,
-                                              AFR_UNLOCK_OP, basename, i);
-
-                        STACK_WIND_COOKIE (frame, afr_unlock_entrylk_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->entrylk,
-                                           this->name,
-                                           loc, basename,
-                                           ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-
-                        if (!--call_count)
-                                break;
-
-                }
-        }
-
-out:
-        return 0;
-
-}
-
-
-static int
-afr_post_unlock_higher_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.done (frame, this);
-        return 0;
-}
-
-static int
-afr_post_unlock_lower_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_local_t         *local       = NULL;
-        loc_t               *lower       = NULL;
-        loc_t               *higher      = NULL;
-        const char          *higher_name = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        lower = lower_path (&local->transaction.parent_loc,
-                            local->transaction.basename,
-                            &local->transaction.new_parent_loc,
-                            local->transaction.new_basename);
-
-        higher = (lower == &local->transaction.parent_loc ?
-                  &local->transaction.new_parent_loc :
-                  &local->transaction.parent_loc);
-
-        higher_name = (higher == &local->transaction.parent_loc ?
-                       local->transaction.basename :
-                       local->transaction.new_basename);
-
-        if (__is_higher_locked (frame, this)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "unlocking higher");
-                int_lock->lk_basename = higher_name;
-                int_lock->lk_loc      = higher;
-                int_lock->lock_cbk    = afr_post_unlock_higher_cbk;
-
-                afr_unlock_entrylk (frame, this);
-        } else
-                local->transaction.done (frame, this);
-
-        return 0;
-}
-
-static int
-afr_rename_unlock (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_local_t         *local       = NULL;
-        loc_t               *lower       = NULL;
-        const char          *lower_name  = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        lower = lower_path (&local->transaction.parent_loc,
-                            local->transaction.basename,
-                            &local->transaction.new_parent_loc,
-                            local->transaction.new_basename);
-
-        lower_name = (lower == &local->transaction.parent_loc ?
-                      local->transaction.basename :
-                      local->transaction.new_basename);
-
-        if (__is_lower_locked (frame, this)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "unlocking lower");
-                int_lock->lk_basename = lower_name;
-                int_lock->lk_loc      = lower;
-                int_lock->lock_cbk    = afr_post_unlock_lower_cbk;
-
-                afr_unlock_lower_entrylk (frame, this);
-        } else
-                afr_post_unlock_lower_cbk (frame, this);
-
-        return 0;
-}
-
-static int
-afr_rename_transaction (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        return (local->transaction.type ==
-                AFR_ENTRY_RENAME_TRANSACTION);
-
-}
-
 int32_t
 afr_unlock (call_frame_t *frame, xlator_t *this)
 {
@@ -1767,10 +1635,8 @@ afr_unlock (call_frame_t *frame, xlator_t *this)
                 if (is_afr_lock_transaction (local))
                         afr_unlock_inodelk (frame, this);
                 else
-                        if (!afr_rename_transaction (frame, this))
-                                afr_unlock_entrylk (frame, this);
-                        else
-                                afr_rename_unlock (frame, this);
+                        afr_unlock_entrylk (frame, this);
+
         } else {
                 if (is_afr_lock_selfheal (local))
                         afr_unlock_inodelk (frame, this);
@@ -1939,10 +1805,12 @@ out:
 
 int32_t
 afr_get_locks_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock);
+                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                      dict_t *xdata);
 int32_t
 afr_recover_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                      dict_t *xdata)
 {
         afr_local_t   *local = NULL;
         afr_private_t *priv  = NULL;
@@ -1966,7 +1834,7 @@ afr_recover_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                            (void *) (long) source_child,
                            priv->children[source_child],
                            priv->children[source_child]->fops->lk,
-                           local->fd, F_GETLK_FD, &flock);
+                           local->fd, F_GETLK_FD, &flock, NULL);
 
         return 0;
 
@@ -1994,7 +1862,7 @@ afr_recover_lock (call_frame_t *frame, xlator_t *this,
                            (void *) (long) lock_recovery_child,
                            priv->children[lock_recovery_child],
                            priv->children[lock_recovery_child]->fops->lk,
-                           local->fd, F_SETLK, flock);
+                           local->fd, F_SETLK, flock, NULL);
 
         return 0;
 }
@@ -2012,7 +1880,8 @@ is_afr_lock_eol (struct gf_flock *lock)
 
 int32_t
 afr_get_locks_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                      dict_t *xdata)
 {
         if (op_ret) {
                 gf_log (this->name, GF_LOG_INFO,
@@ -2072,7 +1941,7 @@ afr_lock_recovery (call_frame_t *frame, xlator_t *this)
                            (void *) (long) source_child,
                            priv->children[source_child],
                            priv->children[source_child]->fops->lk,
-                           local->fd, F_GETLK_FD, &flock);
+                           local->fd, F_GETLK_FD, &flock, NULL);
 
 out:
         return ret;
@@ -2100,7 +1969,8 @@ out:
 
 int32_t
 afr_lock_recovery_preopen_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                               int32_t op_ret, int32_t op_errno, fd_t *fd)
+                               int32_t op_ret, int32_t op_errno, fd_t *fd,
+                               dict_t *xdata)
 {
         int32_t child_index = (long )cookie;
         int ret = 0;
@@ -2172,8 +2042,7 @@ afr_lock_recovery_preopen (call_frame_t *frame, xlator_t *this)
                            (void *)(long) child_index,
                            priv->children[child_index],
                            priv->children[child_index]->fops->open,
-                           &loc, fdctx->flags, local->fd,
-                           fdctx->wbflags);
+                           &loc, fdctx->flags, local->fd, NULL);
 
         return 0;
 }
@@ -2268,27 +2137,38 @@ out:
         return ret;
 }
 
-void
-afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src,
+int
+afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src, char *dom,
                           unsigned int child_count)
 {
-        afr_local_t *dst_local = NULL;
-        afr_local_t *src_local = NULL;
-        afr_internal_lock_t *dst_lock = NULL;
-        afr_internal_lock_t *src_lock = NULL;
+        afr_local_t         *dst_local   = NULL;
+        afr_local_t         *src_local   = NULL;
+        afr_internal_lock_t *dst_lock    = NULL;
+        afr_internal_lock_t *src_lock    = NULL;
+        afr_inodelk_t       *dst_inodelk = NULL;
+        afr_inodelk_t       *src_inodelk = NULL;
+        int                 ret = -1;
 
-        dst_local = dst->local;
-        dst_lock  = &dst_local->internal_lock;
         src_local = src->local;
         src_lock  = &src_local->internal_lock;
-        if (src_lock->inode_locked_nodes) {
-                memcpy (dst_lock->inode_locked_nodes,
-                        src_lock->inode_locked_nodes,
-                        sizeof (*dst_lock->inode_locked_nodes) * child_count);
-                memset (src_lock->inode_locked_nodes, 0,
-                        sizeof (*src_lock->inode_locked_nodes) * child_count);
+        src_inodelk = afr_get_inodelk (src_lock, dom);
+        dst_local = dst->local;
+        dst_lock  = &dst_local->internal_lock;
+        dst_inodelk = afr_get_inodelk (dst_lock, dom);
+        if (!dst_inodelk || !src_inodelk)
+                goto out;
+        if (src_inodelk->locked_nodes) {
+                memcpy (dst_inodelk->locked_nodes, src_inodelk->locked_nodes,
+                        sizeof (*dst_inodelk->locked_nodes) * child_count);
+                memset (src_inodelk->locked_nodes, 0,
+                        sizeof (*src_inodelk->locked_nodes) * child_count);
         }
 
-        dst_lock->inodelk_lock_count = src_lock->inodelk_lock_count;
-        src_lock->inodelk_lock_count = 0;
+        dst_lock->transaction_lk_type = src_lock->transaction_lk_type;
+        dst_lock->selfheal_lk_type    = src_lock->selfheal_lk_type;
+        dst_inodelk->lock_count = src_inodelk->lock_count;
+        src_inodelk->lock_count = 0;
+        ret = 0;
+out:
+        return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-mem-types.h b/xlators/cluster/afr/src/afr-mem-types.h
index f5292b3cc..73594f265 100644
--- a/xlators/cluster/afr/src/afr-mem-types.h
+++ b/xlators/cluster/afr/src/afr-mem-types.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -50,6 +41,10 @@ enum gf_afr_mem_types_ {
         gf_afr_mt_shd_event_t,
         gf_afr_mt_time_t,
         gf_afr_mt_pos_data_t,
+        gf_afr_mt_reply_t,
+        gf_afr_mt_stats_t,
+        gf_afr_mt_shd_crawl_event_t,
+        gf_afr_mt_uint64_t,
         gf_afr_mt_end
 };
 #endif
diff --git a/xlators/cluster/afr/src/afr-open.c b/xlators/cluster/afr/src/afr-open.c
index a203a36f9..643a5d692 100644
--- a/xlators/cluster/afr/src/afr-open.c
+++ b/xlators/cluster/afr/src/afr-open.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -123,7 +114,7 @@ out:
 int
 afr_open_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf)
+                        struct iatt *postbuf, dict_t *xdata)
 {
         afr_local_t * local = frame->local;
         afr_private_t *priv = NULL;
@@ -132,7 +123,7 @@ afr_open_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (afr_open_only_data_self_heal (priv->data_self_heal))
                 afr_perform_data_self_heal (frame, this);
         AFR_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
-                          local->fd);
+                          local->fd, xdata);
         return 0;
 }
 
@@ -140,7 +131,7 @@ afr_open_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 afr_open_cbk (call_frame_t *frame, void *cookie,
               xlator_t *this, int32_t op_ret, int32_t op_errno,
-              fd_t *fd)
+              fd_t *fd, dict_t *xdata)
 {
         afr_local_t *  local       = NULL;
         int            ret         = 0;
@@ -162,8 +153,7 @@ afr_open_cbk (call_frame_t *frame, void *cookie,
                         local->success_count++;
 
                         ret = afr_child_fd_ctx_set (this, fd, child_index,
-                                                    local->cont.open.flags,
-                                                    local->cont.open.wbflags);
+                                                    local->cont.open.flags);
                         if (ret) {
                                 local->op_ret = -1;
                                 local->op_errno = -ret;
@@ -181,12 +171,12 @@ unlock:
                     && (local->op_ret >= 0)) {
                         STACK_WIND (frame, afr_open_ftruncate_cbk,
                                     this, this->fops->ftruncate,
-                                    fd, 0);
+                                    fd, 0, NULL);
                 } else {
                         if (afr_open_only_data_self_heal (priv->data_self_heal))
                                 afr_perform_data_self_heal (frame, this);
                         AFR_STACK_UNWIND (open, frame, local->op_ret,
-                                          local->op_errno, local->fd);
+                                          local->op_errno, local->fd, xdata);
                 }
         }
 
@@ -195,7 +185,7 @@ unlock:
 
 int
 afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, int32_t wbflags)
+          fd_t *fd, dict_t *xdata)
 {
         afr_private_t * priv       = NULL;
         afr_local_t *   local      = NULL;
@@ -236,7 +226,6 @@ afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
         loc_copy (&local->loc, loc);
 
         local->cont.open.flags   = flags;
-        local->cont.open.wbflags = wbflags;
 
         local->fd = fd_ref (fd);
 
@@ -245,7 +234,7 @@ afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
                         STACK_WIND_COOKIE (frame, afr_open_cbk, (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->open,
-                                           loc, wind_flags, fd, wbflags);
+                                           loc, wind_flags, fd, xdata);
 
                         if (!--call_count)
                                 break;
@@ -255,192 +244,131 @@ afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (open, frame, -1, op_errno, fd);
+                AFR_STACK_UNWIND (open, frame, -1, op_errno, fd, xdata);
 
         return 0;
 }
 
-//NOTE: this function should be called with holding the lock on
-//fd to which fd_ctx belongs
-void
-afr_get_resumable_calls (xlator_t *this, afr_fd_ctx_t *fd_ctx,
-                         struct list_head *list)
-{
-        afr_fd_paused_call_t *paused_call = NULL;
-        afr_fd_paused_call_t *tmp = NULL;
-        afr_local_t           *call_local  = NULL;
-        afr_private_t         *priv        = NULL;
-        int                    i = 0;
-        gf_boolean_t           call = _gf_false;
-
-        priv = this->private;
-        list_for_each_entry_safe (paused_call, tmp, &fd_ctx->paused_calls,
-                                  call_list) {
-                call = _gf_true;
-                call_local = paused_call->frame->local;
-                for (i = 0; i < priv->child_count; i++) {
-                        if (call_local->child_up[i] &&
-                            (fd_ctx->opened_on[i] == AFR_FD_OPENING))
-                                call = _gf_false;
-                }
-
-                if (call) {
-                        list_del_init (&paused_call->call_list);
-                        list_add (&paused_call->call_list, list);
-                }
-        }
-}
-
-void
-afr_resume_calls (xlator_t *this, struct list_head *list)
-{
-        afr_fd_paused_call_t *paused_call = NULL;
-        afr_fd_paused_call_t *tmp = NULL;
-        afr_local_t           *call_local  = NULL;
-
-        list_for_each_entry_safe (paused_call, tmp, list, call_list) {
-                list_del_init (&paused_call->call_list);
-                call_local = paused_call->frame->local;
-                call_local->fop_call_continue (paused_call->frame, this);
-                GF_FREE (paused_call);
-        }
-}
-
 int
 afr_openfd_fix_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, fd_t *fd)
+                         int32_t op_ret, int32_t op_errno, fd_t *fd,
+                         dict_t *xdata)
 {
-        afr_local_t           *local       = NULL;
-        afr_private_t         *priv        = NULL;
-        afr_fd_ctx_t          *fd_ctx      = NULL;
-        int                    call_count  = 0;
-        int                    child_index = (long) cookie;
-        struct list_head       paused_calls = {0};
-        gf_boolean_t           fop_paused = _gf_false;
-        fd_t                   *local_fd  = NULL;
+        afr_local_t   *local      = NULL;
+        afr_private_t *priv       = NULL;
+        afr_fd_ctx_t  *fd_ctx     = NULL;
+        int           call_count  = 0;
+        int           child_index = (long) cookie;
 
         priv     = this->private;
         local    = frame->local;
-        fop_paused = local->fop_paused;
 
         if (op_ret >= 0) {
-                gf_log (this->name, GF_LOG_INFO, "fd for %s opened "
+                gf_log (this->name, GF_LOG_DEBUG, "fd for %s opened "
                         "successfully on subvolume %s", local->loc.path,
                         priv->children[child_index]->name);
+        } else {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to open %s "
+                        "on subvolume %s", local->loc.path,
+                        priv->children[child_index]->name);
         }
 
-        local_fd = fd_ref (local->fd);
-        call_count = afr_frame_return (frame);
-        //Note: Do not access any thing using the frame outside call_count 0
-
-        //Note: No frame locking needed for this block of code
-        fd_ctx = afr_fd_ctx_get (local_fd, this);
+        fd_ctx = afr_fd_ctx_get (local->fd, this);
         if (!fd_ctx) {
                 gf_log (this->name, GF_LOG_WARNING,
-                        "failed to get fd context, %p", local_fd);
+                        "failed to get fd context, %p", local->fd);
                 goto out;
         }
 
-        LOCK (&local_fd->lock);
+        LOCK (&local->fd->lock);
         {
                 if (op_ret >= 0) {
                         fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
                 } else {
                         fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
                 }
-                if (call_count == 0) {
-                        INIT_LIST_HEAD (&paused_calls);
-                        afr_get_resumable_calls (this, fd_ctx, &paused_calls);
-                }
         }
-        UNLOCK (&local_fd->lock);
+        UNLOCK (&local->fd->lock);
 out:
-        if (call_count == 0) {
-                afr_resume_calls (this, &paused_calls);
-                //If the fop is paused then resume_calls will continue the fop
-                if (fop_paused)
-                        goto done;
-
-                if (local->fop_call_continue)
-                        local->fop_call_continue (frame, this);
-                else
-                        AFR_STACK_DESTROY (frame);
-        }
+        call_count = afr_frame_return (frame);
+        if (call_count == 0)
+                AFR_STACK_DESTROY (frame);
 
-done:
-        fd_unref (local_fd);
         return 0;
 }
 
-int
-afr_fix_open (call_frame_t *frame, xlator_t *this, afr_fd_ctx_t *fd_ctx,
-              int need_open_count, int *need_open)
+void
+afr_fix_open (xlator_t *this, fd_t *fd, size_t need_open_count, int *need_open)
 {
-        afr_local_t       *local = NULL;
-        afr_private_t     *priv  = NULL;
-        int               i      = 0;
-        call_frame_t      *open_frame = NULL;
-        afr_local_t      *open_local = NULL;
-        int               ret    = -1;
-        ia_type_t         ia_type = IA_INVAL;
-        int32_t           op_errno = 0;
-
-        GF_ASSERT (fd_ctx);
-        GF_ASSERT (need_open_count > 0);
-        GF_ASSERT (need_open);
+        afr_private_t *priv    = NULL;
+        int           i        = 0;
+        call_frame_t  *frame   = NULL;
+        afr_local_t   *local   = NULL;
+        int           ret      = -1;
+        int32_t       op_errno = 0;
+        afr_fd_ctx_t  *fd_ctx  = NULL;
 
-        local = frame->local;
         priv  = this->private;
-        if (!local->fop_call_continue) {
-                open_frame = copy_frame (frame);
-                if (!open_frame) {
-                        ret = -ENOMEM;
-                        goto out;
-                }
-                AFR_LOCAL_ALLOC_OR_GOTO (open_frame->local, out);
-                open_local = open_frame->local;
-                ret = afr_local_init (open_local, priv, &op_errno);
-                if (ret < 0)
-                        goto out;
-                loc_copy (&open_local->loc, &local->loc);
-                open_local->fd = fd_ref (local->fd);
-        } else {
-                ret = 0;
-                open_frame = frame;
-                open_local = local;
+
+        if (!afr_is_fd_fixable (fd) || !need_open || !need_open_count)
+                goto out;
+
+        fd_ctx = afr_fd_ctx_get (fd, this);
+        if (!fd_ctx) {
+                ret = -1;
+                goto out;
+        }
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame) {
+                ret = -1;
+                goto out;
         }
 
-        open_local->call_count = need_open_count;
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        local->loc.inode = inode_ref (fd->inode);
+        ret = loc_path (&local->loc, NULL);
+        if (ret < 0)
+                goto out;
+
+        local->fd = fd_ref (fd);
+        local->call_count = need_open_count;
 
-        gf_log (this->name, GF_LOG_DEBUG, "need open count: %d",
+        gf_log (this->name, GF_LOG_DEBUG, "need open count: %zd",
                 need_open_count);
 
-        ia_type = open_local->fd->inode->ia_type;
-        GF_ASSERT (ia_type != IA_INVAL);
         for (i = 0; i < priv->child_count; i++) {
                 if (!need_open[i])
                         continue;
-                if (IA_IFDIR == ia_type) {
+
+                if (IA_IFDIR == fd->inode->ia_type) {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "opening fd for dir %s on subvolume %s",
                                 local->loc.path, priv->children[i]->name);
 
-                        STACK_WIND_COOKIE (open_frame, afr_openfd_fix_open_cbk,
+                        STACK_WIND_COOKIE (frame, afr_openfd_fix_open_cbk,
                                            (void*) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->opendir,
-                                           &open_local->loc, open_local->fd);
+                                           &local->loc, local->fd,
+                                           NULL);
                 } else {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "opening fd for file %s on subvolume %s",
                                 local->loc.path, priv->children[i]->name);
 
-                        STACK_WIND_COOKIE (open_frame, afr_openfd_fix_open_cbk,
+                        STACK_WIND_COOKIE (frame, afr_openfd_fix_open_cbk,
                                            (void *)(long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->open,
-                                           &open_local->loc, fd_ctx->flags,
-                                           open_local->fd, fd_ctx->wbflags);
+                                           &local->loc,
+                                           fd_ctx->flags & (~O_TRUNC),
+                                           local->fd, NULL);
                 }
 
         }
@@ -448,8 +376,7 @@ afr_fix_open (call_frame_t *frame, xlator_t *this, afr_fd_ctx_t *fd_ctx,
         ret = 0;
 out:
         if (op_errno)
-                ret = -op_errno;
-        if (ret && open_frame)
-                AFR_STACK_DESTROY (open_frame);
-        return ret;
+                ret = -1; //For handling ALLOC_OR_GOTO
+        if (ret && frame)
+                AFR_STACK_DESTROY (frame);
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-algorithm.c b/xlators/cluster/afr/src/afr-self-heal-algorithm.c
index 629822a8e..83846f152 100644
--- a/xlators/cluster/afr/src/afr-self-heal-algorithm.c
+++ b/xlators/cluster/afr/src/afr-self-heal-algorithm.c
@@ -1,23 +1,15 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
+#include <openssl/md5.h>
 #include "glusterfs.h"
 #include "afr.h"
 #include "xlator.h"
@@ -33,7 +25,6 @@
 #include "compat-errno.h"
 #include "compat.h"
 #include "byte-order.h"
-#include "md5.h"
 
 #include "afr-transaction.h"
 #include "afr-self-heal.h"
@@ -72,8 +63,7 @@ sh_private_cleanup (call_frame_t *frame, xlator_t *this)
         sh    = &local->self_heal;
 
         sh_priv = sh->private;
-        if (sh_priv)
-                GF_FREE (sh_priv);
+        GF_FREE (sh_priv);
 }
 
 static int
@@ -110,10 +100,10 @@ sh_loop_driver_done (call_frame_t *sh_frame, xlator_t *this,
         }
 
         sh_private_cleanup (sh_frame, this);
-        if (sh->op_failed) {
+        if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
                 GF_ASSERT (!last_loop_frame);
                 //loop_finish should have happened and the old_loop should be NULL
-                gf_log (this->name, GF_LOG_INFO,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "self-heal aborting on %s",
                         local->loc.path);
 
@@ -121,10 +111,10 @@ sh_loop_driver_done (call_frame_t *sh_frame, xlator_t *this,
         } else {
                 GF_ASSERT (last_loop_frame);
                 if (diff_blocks == total_blocks) {
-                        gf_log (this->name, GF_LOG_INFO, "full self-heal "
+                        gf_log (this->name, GF_LOG_DEBUG, "full self-heal "
                                 "completed on %s",local->loc.path);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
+                        gf_log (this->name, GF_LOG_DEBUG,
                                 "diff self-heal on %s: completed. "
                                 "(%d blocks of %d were different (%.2f%%))",
                                 local->loc.path, diff_blocks, total_blocks,
@@ -153,7 +143,7 @@ sh_loop_finish (call_frame_t *loop_frame, xlator_t *this)
         }
 
         if (loop_sh && loop_sh->data_lock_held) {
-                afr_sh_data_unlock (loop_frame, this,
+                afr_sh_data_unlock (loop_frame, this, this->name,
                                     sh_destroy_frame);
         } else {
                 sh_destroy_frame (loop_frame, this);
@@ -174,7 +164,7 @@ sh_loop_lock_success (call_frame_t *loop_frame, xlator_t *this)
         sh_loop_finish (loop_sh->old_loop_frame, this);
         loop_sh->old_loop_frame = NULL;
 
-        gf_log (this->name, GF_LOG_DEBUG, "Aquired lock for range %"PRIu64
+        gf_log (this->name, GF_LOG_DEBUG, "Acquired lock for range %"PRIu64
                 " %"PRIu64, loop_sh->offset, loop_sh->block_size);
         loop_sh->data_lock_held = _gf_true;
         loop_sh->sh_data_algo_start (loop_frame, this);
@@ -224,7 +214,7 @@ sh_loop_frame_create (call_frame_t *sh_frame, xlator_t *this,
                 goto out;
         //We want the frame to have same lk_owner as sh_frame
         //so that locks translator allows conflicting locks
-        new_loop_local = afr_local_copy (local, this);
+        new_loop_local = afr_self_heal_local_init (local, this);
         if (!new_loop_local)
                 goto out;
         new_loop_frame->local = new_loop_local;
@@ -239,7 +229,7 @@ sh_loop_frame_create (call_frame_t *sh_frame, xlator_t *this,
                                                gf_afr_mt_char);
         if (!new_loop_sh->write_needed)
                 goto out;
-        new_loop_sh->checksum = GF_CALLOC (priv->child_count, MD5_DIGEST_LEN,
+        new_loop_sh->checksum = GF_CALLOC (priv->child_count, MD5_DIGEST_LENGTH,
                                            gf_afr_mt_uint8_t);
         if (!new_loop_sh->checksum)
                 goto out;
@@ -283,10 +273,10 @@ sh_loop_start (call_frame_t *sh_frame, xlator_t *this, off_t offset,
         new_loop_sh->offset = offset;
         new_loop_sh->block_size = sh->block_size;
         afr_sh_data_lock (new_loop_frame, this, offset, new_loop_sh->block_size,
-                          sh_loop_lock_success, sh_loop_lock_failure);
+                          _gf_true, this->name, sh_loop_lock_success, sh_loop_lock_failure);
         return 0;
 out:
-        sh->op_failed = 1;
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
         if (old_loop_frame)
                 sh_loop_finish (old_loop_frame, this);
         sh_loop_return (sh_frame, this, new_loop_frame, -1, ENOMEM);
@@ -317,8 +307,9 @@ sh_loop_driver (call_frame_t *sh_frame, xlator_t *this,
                         sh_priv->loops_running--;
                 offset = sh_priv->offset;
                 block_size = sh->block_size;
-                while ((!sh->eof_reached) && (0 == sh->op_failed) &&
-                       (sh_priv->loops_running < priv->data_self_heal_window_size)
+                while ((!sh->eof_reached) &&
+                       (!is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) &&
+                     (sh_priv->loops_running < priv->data_self_heal_window_size)
                        && (sh_priv->offset < sh->file_size)) {
 
                         loop++;
@@ -337,7 +328,8 @@ sh_loop_driver (call_frame_t *sh_frame, xlator_t *this,
         if (0 == loop) {
                 //loop finish does unlock, but the erasing of the pending
                 //xattrs needs to happen before that so do not finish the loop
-                if (is_driver_done && !sh->op_failed)
+                if (is_driver_done &&
+                    !is_self_heal_failed (sh, AFR_CHECK_SPECIFIC))
                         goto driver_done;
                 if (old_loop_frame) {
                         sh_loop_finish (old_loop_frame, this);
@@ -348,7 +340,7 @@ sh_loop_driver (call_frame_t *sh_frame, xlator_t *this,
         //If we have more loops to form we should finish previous loop after
         //the next loop lock
         while (loop--) {
-                if (sh->op_failed) {
+                if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
                         // op failed in other loop, stop spawning more loops
                         if (old_loop_frame) {
                                 sh_loop_finish (old_loop_frame, this);
@@ -394,7 +386,7 @@ sh_loop_return (call_frame_t *sh_frame, xlator_t *this, call_frame_t *loop_frame
         }
 
         if (op_ret == -1) {
-                sh->op_failed = 1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 afr_sh_set_error (sh, op_errno);
                 if (loop_frame) {
                         sh_loop_finish (loop_frame, this);
@@ -410,7 +402,7 @@ sh_loop_return (call_frame_t *sh_frame, xlator_t *this, call_frame_t *loop_frame
 static int
 sh_loop_write_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                   struct iatt *postbuf)
+                   struct iatt *postbuf, dict_t *xdata)
 {
         afr_private_t *             priv        = NULL;
         afr_local_t *               loop_local    = NULL;
@@ -442,13 +434,22 @@ sh_loop_write_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
                         priv->children[child_index]->name,
                         strerror (op_errno));
 
-                sh->op_failed = 1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 afr_sh_set_error (loop_sh, op_errno);
+        } else if (op_ret < loop_local->cont.writev.vector->iov_len) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "incomplete write to %s on subvolume %s "
+                        "(expected %lu, returned %d)", sh_local->loc.path,
+                        priv->children[child_index]->name,
+                        loop_local->cont.writev.vector->iov_len, op_ret);
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
         }
 
         call_count = afr_frame_return (loop_frame);
 
         if (call_count == 0) {
+		iobref_unref(loop_local->cont.writev.iobref);
+
                 sh_loop_return (sh_frame, this, loop_frame,
                                 loop_sh->op_ret, loop_sh->op_errno);
         }
@@ -456,12 +457,41 @@ sh_loop_write_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
         return 0;
 }
 
+static void
+sh_prune_writes_needed (call_frame_t *sh_frame, call_frame_t *loop_frame,
+                        afr_private_t *priv)
+{
+        afr_local_t     *sh_local     = NULL;
+        afr_self_heal_t *sh           = NULL;
+        afr_local_t     *loop_local   = NULL;
+        afr_self_heal_t *loop_sh      = NULL;
+        int             i             = 0;
+
+        sh_local   = sh_frame->local;
+        sh         = &sh_local->self_heal;
+
+        if (!strcmp (sh->algo->name, "diff"))
+                return;
+
+        loop_local = loop_frame->local;
+        loop_sh    = &loop_local->self_heal;
+
+        /* full self-heal guarantees there exists atleast 1 file with size 0
+         * That means for other files we can preserve holes that come after
+         * its size before 'trim'
+         */
+        for (i = 0; i < priv->child_count; i++) {
+                if (loop_sh->write_needed[i] &&
+                    ((loop_sh->offset + 1) > sh->buf[i].ia_size))
+                        loop_sh->write_needed[i] = 0;
+        }
+}
 
 static int
 sh_loop_read_cbk (call_frame_t *loop_frame, void *cookie,
                   xlator_t *this, int32_t op_ret, int32_t op_errno,
                   struct iovec *vector, int32_t count, struct iatt *buf,
-                  struct iobref *iobref)
+                  struct iobref *iobref, dict_t *xdata)
 {
         afr_private_t *               priv       = NULL;
         afr_local_t *                 loop_local   = NULL;
@@ -486,7 +516,7 @@ sh_loop_read_cbk (call_frame_t *loop_frame, void *cookie,
 
         if (op_ret <= 0) {
                 if (op_ret < 0) {
-                        sh->op_failed = 1;
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                         gf_log (this->name, GF_LOG_ERROR, "read failed on %d "
                                 "for %s reason :%s", sh->source,
                                 sh_local->loc.path, strerror (errno));
@@ -499,18 +529,26 @@ sh_loop_read_cbk (call_frame_t *loop_frame, void *cookie,
                 goto out;
         }
 
-        if (loop_sh->file_has_holes && iov_0filled (vector, count) == 0) {
-                        gf_log (this->name, GF_LOG_DEBUG, "0 filled block");
-                        sh_loop_return (sh_frame, this, loop_frame,
-                                        op_ret, op_errno);
-                        goto out;
-        }
+        if (loop_sh->file_has_holes && iov_0filled (vector, count) == 0)
+                sh_prune_writes_needed (sh_frame, loop_frame, priv);
 
         call_count = sh_number_of_writes_needed (loop_sh->write_needed,
                                                  priv->child_count);
-        GF_ASSERT (call_count > 0);
+        if (call_count == 0) {
+                sh_loop_return (sh_frame, this, loop_frame, 0, 0);
+                goto out;
+        }
+
         loop_local->call_count = call_count;
 
+	/*
+	 * We only really need the request size at the moment, but the buffer
+	 * is required if we want to issue a retry in the event of a short write.
+	 * Therefore, we duplicate the vector and ref the iobref here...
+	 */
+	loop_local->cont.writev.vector = iov_dup(vector, count);
+	loop_local->cont.writev.iobref = iobref_ref(iobref);
+
         for (i = 0; i < priv->child_count; i++) {
                 if (!loop_sh->write_needed[i])
                         continue;
@@ -519,7 +557,7 @@ sh_loop_read_cbk (call_frame_t *loop_frame, void *cookie,
                                    priv->children[i],
                                    priv->children[i]->fops->writev,
                                    loop_sh->healing_fd, vector, count,
-                                   loop_sh->offset, 0, iobref);
+                                   loop_sh->offset, 0, iobref, NULL);
 
                 if (!--call_count)
                         break;
@@ -546,7 +584,7 @@ sh_loop_read (call_frame_t *loop_frame, xlator_t *this)
                            priv->children[loop_sh->source],
                            priv->children[loop_sh->source]->fops->readv,
                            loop_sh->healing_fd, loop_sh->block_size,
-                           loop_sh->offset, 0);
+                           loop_sh->offset, 0, NULL);
 
         return 0;
 }
@@ -555,7 +593,8 @@ sh_loop_read (call_frame_t *loop_frame, xlator_t *this)
 static int
 sh_diff_checksum_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno,
-                      uint32_t weak_checksum, uint8_t *strong_checksum)
+                      uint32_t weak_checksum, uint8_t *strong_checksum,
+                      dict_t *xdata)
 {
         afr_private_t                 *priv         = NULL;
         afr_local_t                   *loop_local   = NULL;
@@ -587,10 +626,10 @@ sh_diff_checksum_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
                         "checksum on %s failed on subvolume %s (%s)",
                         sh_local->loc.path, priv->children[child_index]->name,
                         strerror (op_errno));
-                sh->op_failed = 1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
         } else {
-                memcpy (loop_sh->checksum + child_index * MD5_DIGEST_LEN,
-                        strong_checksum, MD5_DIGEST_LEN);
+                memcpy (loop_sh->checksum + child_index * MD5_DIGEST_LENGTH,
+                        strong_checksum, MD5_DIGEST_LENGTH);
         }
 
         call_count = afr_frame_return (loop_frame);
@@ -600,9 +639,9 @@ sh_diff_checksum_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
                         if (sh->sources[i] || !sh_local->child_up[i])
                                 continue;
 
-                        if (memcmp (loop_sh->checksum + (i * MD5_DIGEST_LEN),
-                                    loop_sh->checksum + (sh->source * MD5_DIGEST_LEN),
-                                    MD5_DIGEST_LEN)) {
+                        if (memcmp (loop_sh->checksum + (i * MD5_DIGEST_LENGTH),
+                                    loop_sh->checksum + (sh->source * MD5_DIGEST_LENGTH),
+                                    MD5_DIGEST_LENGTH)) {
                                 /*
                                   Checksums differ, so this block
                                   must be written to this sink
@@ -625,7 +664,8 @@ sh_diff_checksum_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
                 }
                 UNLOCK (&sh_priv->lock);
 
-                if (write_needed && !sh->op_failed) {
+                if (write_needed &&
+                    !is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
                         sh_loop_read (loop_frame, this);
                 } else {
                         sh_loop_return (sh_frame, this, loop_frame,
@@ -658,7 +698,7 @@ sh_diff_checksum (call_frame_t *loop_frame, xlator_t *this)
                            priv->children[loop_sh->source],
                            priv->children[loop_sh->source]->fops->rchecksum,
                            loop_sh->healing_fd,
-                           loop_sh->offset, loop_sh->block_size);
+                           loop_sh->offset, loop_sh->block_size, NULL);
 
         for (i = 0; i < priv->child_count; i++) {
                 if (loop_sh->sources[i] || !loop_local->child_up[i])
@@ -669,7 +709,7 @@ sh_diff_checksum (call_frame_t *loop_frame, xlator_t *this)
                                    priv->children[i],
                                    priv->children[i]->fops->rchecksum,
                                    loop_sh->healing_fd,
-                                   loop_sh->offset, loop_sh->block_size);
+                                   loop_sh->offset, loop_sh->block_size, NULL);
 
                 if (!--call_count)
                         break;
@@ -714,14 +754,15 @@ out:
         return sh_priv;
 }
 
-void
-afr_sh_transfer_lock (call_frame_t *dst, call_frame_t *src,
+int
+afr_sh_transfer_lock (call_frame_t *dst, call_frame_t *src, char *dom,
                       unsigned int child_count)
 {
         afr_local_t             *dst_local   = NULL;
         afr_self_heal_t         *dst_sh      = NULL;
         afr_local_t             *src_local   = NULL;
         afr_self_heal_t         *src_sh      = NULL;
+        int                     ret          = -1;
 
         dst_local = dst->local;
         dst_sh = &dst_local->self_heal;
@@ -729,9 +770,12 @@ afr_sh_transfer_lock (call_frame_t *dst, call_frame_t *src,
         src_sh = &src_local->self_heal;
         GF_ASSERT (src_sh->data_lock_held);
         GF_ASSERT (!dst_sh->data_lock_held);
-        afr_lk_transfer_datalock (dst, src, child_count);
+        ret = afr_lk_transfer_datalock (dst, src, dom, child_count);
+        if (ret)
+                return ret;
         src_sh->data_lock_held = _gf_false;
         dst_sh->data_lock_held = _gf_true;
+        return 0;
 }
 
 int
@@ -753,7 +797,10 @@ afr_sh_start_loops (call_frame_t *sh_frame, xlator_t *this,
         ret = sh_loop_frame_create (sh_frame, this, NULL, &first_loop_frame);
         if (ret)
                 goto out;
-        afr_sh_transfer_lock (first_loop_frame, sh_frame, priv->child_count);
+        ret = afr_sh_transfer_lock (first_loop_frame, sh_frame, this->name,
+                                    priv->child_count);
+        if (ret)
+                goto out;
         sh->private = afr_sh_priv_init ();
         if (!sh->private) {
                 ret = -1;
@@ -763,7 +810,7 @@ afr_sh_start_loops (call_frame_t *sh_frame, xlator_t *this,
         ret = 0;
 out:
         if (ret) {
-                sh->op_failed = 1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 sh_loop_driver_done (sh_frame, this, NULL);
         }
         return 0;
diff --git a/xlators/cluster/afr/src/afr-self-heal-algorithm.h b/xlators/cluster/afr/src/afr-self-heal-algorithm.h
index 04d8e8a6c..6b20789b1 100644
--- a/xlators/cluster/afr/src/afr-self-heal-algorithm.h
+++ b/xlators/cluster/afr/src/afr-self-heal-algorithm.h
@@ -1,26 +1,16 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __AFR_SELF_HEAL_ALGORITHM_H__
 #define __AFR_SELF_HEAL_ALGORITHM_H__
 
-
 typedef int (*afr_sh_algo_fn) (call_frame_t *frame,
                                xlator_t *this);
 
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index f9958e768..ef92b4205 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "glusterfs.h"
@@ -27,6 +18,28 @@
 #include "afr-self-heal.h"
 #include "pump.h"
 
+#define ADD_FMT_STRING(msg, off, sh_str, status, print_log)                 \
+        do {                                                                \
+                if (AFR_SELF_HEAL_NOT_ATTEMPTED != status) {                \
+                        off += snprintf (msg + off, sizeof (msg) - off,     \
+                                         " "sh_str" self heal %s,",         \
+                                         get_sh_completion_status (status));\
+                        print_log = 1;                                      \
+                }                                                           \
+        } while (0)
+
+#define ADD_FMT_STRING_SYNC(msg, off, sh_str, status, print_log)            \
+        do {                                                                \
+                if (AFR_SELF_HEAL_SYNC_BEGIN == status ||                   \
+                    AFR_SELF_HEAL_FAILED == status)  {                      \
+                        off += snprintf (msg + off, sizeof (msg) - off,     \
+                                         " "sh_str" self heal %s,",         \
+                                         get_sh_completion_status (status));\
+                        print_log = 1;                                      \
+                }                                                           \
+        } while (0)
+
+
 void
 afr_sh_reset (call_frame_t *frame, xlator_t *this)
 {
@@ -121,8 +134,8 @@ void
 afr_sh_set_error (afr_self_heal_t *sh, int32_t op_errno)
 {
         sh->op_ret = -1;
-        if (afr_error_more_important (sh->op_errno, op_errno))
-                sh->op_errno = op_errno;
+	sh->op_errno = afr_most_important_error(sh->op_errno, op_errno,
+						_gf_false);
 }
 
 void
@@ -144,13 +157,85 @@ afr_sh_print_pending_matrix (int32_t *pending_matrix[], xlator_t *this)
                         ptr += sprintf (ptr, "%d ", pending_matrix[i][j]);
                 }
                 sprintf (ptr, "]");
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "pending_matrix: %s", buf);
+                gf_log (this->name, GF_LOG_DEBUG, "pending_matrix: %s", buf);
         }
 
         GF_FREE (buf);
 }
 
+char*
+afr_get_pending_matrix_str (int32_t *pending_matrix[], xlator_t *this)
+{
+        afr_private_t *  priv = this->private;
+        char            *buf  = NULL;
+        char            *ptr  = NULL;
+        int              i    = 0;
+        int              j    = 0;
+        int             child_count = priv->child_count;
+        char            *matrix_begin = "[ [ ";
+        char            *matrix_end = "] ]";
+        char            *seperator = "] [ ";
+        int             pending_entry_strlen = 12; //Including space after entry
+        int             matrix_begin_strlen = 0;
+        int             matrix_end_strlen = 0;
+        int             seperator_strlen = 0;
+        int             string_length = 0;
+        char            *msg = "- Pending matrix:  ";
+
+        /*
+         *  for a list of lists of [ [ a b ] [ c d ] ]
+         * */
+
+        matrix_begin_strlen = strlen (matrix_begin);
+        matrix_end_strlen = strlen (matrix_end);
+        seperator_strlen = strlen (seperator);
+        string_length = matrix_begin_strlen + matrix_end_strlen
+                        + (child_count -1) * seperator_strlen
+                        + (child_count * child_count * pending_entry_strlen);
+
+        buf = GF_CALLOC (1, 1 + strlen (msg) + string_length , gf_afr_mt_char);
+        if (!buf)
+                goto out;
+
+        ptr = buf;
+        ptr += sprintf (ptr, "%s", msg);
+        ptr += sprintf (ptr, "%s", matrix_begin);
+        for (i = 0; i < priv->child_count; i++) {
+                for (j = 0; j < priv->child_count; j++) {
+                        ptr += sprintf (ptr, "%d ", pending_matrix[i][j]);
+                }
+                if (i < priv->child_count -1)
+                        ptr += sprintf (ptr, "%s", seperator);
+        }
+
+        ptr += sprintf (ptr, "%s", matrix_end);
+
+out:
+        return buf;
+}
+
+void
+afr_sh_print_split_brain_log (int32_t *pending_matrix[], xlator_t *this,
+                              const char *loc)
+{
+        char *buf      = NULL;
+        char *free_ptr = NULL;
+
+        buf = afr_get_pending_matrix_str (pending_matrix, this);
+        if (buf)
+                free_ptr = buf;
+        else
+                buf = "";
+
+
+        gf_log (this->name, GF_LOG_ERROR, "Unable to self-heal contents of '%s'"
+                " (possible split-brain). Please delete the file from all but "
+                "the preferred subvolume.%s", loc, buf);
+        GF_FREE (free_ptr);
+        return;
+}
+
+
 void
 afr_init_pending_matrix (int32_t **pending_matrix, size_t child_count)
 {
@@ -416,6 +501,8 @@ afr_find_biggest_witness_among_fools (int32_t *witnesses,
 {
         int i               = 0;
         int biggest_witness = -1;
+        int biggest_witness_idx = -1;
+        int biggest_witness_cnt = -1;
 
         GF_ASSERT (witnesses);
         GF_ASSERT (characters);
@@ -425,10 +512,21 @@ afr_find_biggest_witness_among_fools (int32_t *witnesses,
                 if (characters[i].type != AFR_NODE_FOOL)
                         continue;
 
-                if (biggest_witness < witnesses[i])
+                if (biggest_witness < witnesses[i]) {
                         biggest_witness = witnesses[i];
+			biggest_witness_idx = i;
+			biggest_witness_cnt = 1;
+			continue;
+		}
+
+		if (biggest_witness == witnesses[i])
+			biggest_witness_cnt++;
         }
-        return biggest_witness;
+
+	if (biggest_witness_cnt != 1)
+		return -1;
+
+        return biggest_witness_idx;
 }
 
 int
@@ -456,10 +554,84 @@ afr_mark_fool_as_source_by_witness (int32_t *sources, int32_t *witnesses,
         return nsources;
 }
 
+
+int
+afr_mark_fool_as_source_by_idx (int32_t *sources, int child_count, int idx)
+{
+	if (idx >= 0 && idx < child_count) {
+		sources[idx] = 1;
+		return 1;
+	}
+	return 0;
+}
+
+
+static int
+afr_find_largest_file_size (struct iatt *bufs, int32_t *success_children,
+			    int child_count)
+{
+	int idx = -1;
+	int i = -1;
+	int child = -1;
+	uint64_t max_size = 0;
+        uint64_t min_size = 0;
+        int      num_children = 0;
+
+	for (i = 0; i < child_count; i++) {
+		if (success_children[i] == -1)
+			break;
+
+		child = success_children[i];
+		if (bufs[child].ia_size > max_size) {
+			max_size = bufs[child].ia_size;
+			idx = child;
+		}
+
+                if ((num_children == 0) || (bufs[child].ia_size < min_size)) {
+                        min_size = bufs[child].ia_size;
+                }
+
+                num_children++;
+	}
+
+        /* If sizes are same for all of them, finding sources will have to
+         * happen with pending changelog. So return -1
+         */
+        if ((num_children > 1) && (min_size == max_size))
+                return -1;
+	return idx;
+}
+
+
+static int
+afr_find_newest_file (struct iatt *bufs, int32_t *success_children,
+		      int child_count)
+{
+	int idx = -1;
+	int i = -1;
+	int child = -1;
+	uint64_t max_ctime = 0;
+
+	for (i = 0; i < child_count; i++) {
+		if (success_children[i] == -1)
+			break;
+
+		child = success_children[i];
+		if (bufs[child].ia_ctime > max_ctime) {
+			max_ctime = bufs[child].ia_ctime;
+			idx = child;
+		}
+	}
+
+	return idx;
+}
+
+
 static int
 afr_mark_biggest_of_fools_as_source (int32_t *sources, int32_t **pending_matrix,
                                      afr_node_character *characters,
-                                     int child_count)
+				     int32_t *success_children,
+                                     int child_count, struct iatt *bufs)
 {
         int32_t       biggest_witness = 0;
         int           nsources        = 0;
@@ -467,6 +639,11 @@ afr_mark_biggest_of_fools_as_source (int32_t *sources, int32_t **pending_matrix,
 
         GF_ASSERT (child_count > 0);
 
+	biggest_witness = afr_find_largest_file_size (bufs, success_children,
+						      child_count);
+	if (biggest_witness != -1)
+		goto found;
+
         witnesses = GF_CALLOC (child_count, sizeof (*witnesses),
                                gf_afr_mt_int32_t);
         if (NULL == witnesses) {
@@ -479,12 +656,17 @@ afr_mark_biggest_of_fools_as_source (int32_t *sources, int32_t **pending_matrix,
         biggest_witness = afr_find_biggest_witness_among_fools (witnesses,
                                                                 characters,
                                                                 child_count);
-        nsources = afr_mark_fool_as_source_by_witness (sources, witnesses,
-                                                       characters, child_count,
-                                                       biggest_witness);
+	if (biggest_witness != -1)
+		goto found;
+
+	biggest_witness = afr_find_newest_file (bufs, success_children,
+						child_count);
+
+found:
+	nsources = afr_mark_fool_as_source_by_idx (sources, child_count,
+						   biggest_witness);
 out:
-        if (witnesses)
-                GF_FREE (witnesses);
+        GF_FREE (witnesses);
         return nsources;
 }
 
@@ -550,6 +732,78 @@ out:
         return nsources;
 }
 
+int
+afr_get_no_xattr_dir_read_child (xlator_t *this, int32_t *success_children,
+                                 struct iatt *bufs)
+{
+        afr_private_t *priv = NULL;
+        int            i = 0;
+        int            child = -1;
+        int            read_child = -1;
+
+        priv = this->private;
+        for (i = 0; i < priv->child_count; i++) {
+                child = success_children[i];
+                if (child < 0)
+                        break;
+                if (read_child < 0)
+                        read_child = child;
+                else if (bufs[read_child].ia_size < bufs[child].ia_size)
+                        read_child = child;
+        }
+        return read_child;
+}
+
+int
+afr_sh_mark_zero_size_file_as_sink (struct iatt *bufs, int32_t *success_children,
+                                    int child_count, int32_t *sources)
+{
+        int             nsources = 0;
+        int             i = 0;
+        int             child = 0;
+        gf_boolean_t    sink_exists = _gf_false;
+        gf_boolean_t    source_exists = _gf_false;
+        int             source = -1;
+
+        for (i = 0; i < child_count; i++) {
+                child = success_children[i];
+                if (child < 0)
+                        break;
+                if (!bufs[child].ia_size) {
+                        sink_exists = _gf_true;
+                        continue;
+                }
+                if (!source_exists) {
+                        source_exists = _gf_true;
+                        source = child;
+                        continue;
+                }
+                if (bufs[source].ia_size != bufs[child].ia_size) {
+                        nsources = -1;
+                        goto out;
+                }
+        }
+        if (!source_exists && !sink_exists) {
+                nsources = -1;
+                goto out;
+        }
+
+        if (!source_exists || !sink_exists)
+                goto out;
+
+        for (i = 0; i < child_count; i++) {
+                child = success_children[i];
+                if (child < 0)
+                        break;
+                if (bufs[child].ia_size) {
+                        sources[child] = 1;
+                        nsources++;
+                }
+        }
+out:
+        return nsources;
+}
+
 char *
 afr_get_character_str (afr_node_type type)
 {
@@ -715,11 +969,24 @@ afr_mark_sources (xlator_t *this, int32_t *sources, int32_t **pending_matrix,
         afr_find_character_types (characters, pending_matrix, success_children,
                                   child_count);
         if (afr_sh_all_nodes_innocent (characters, child_count)) {
-                if (type == AFR_SELF_HEAL_METADATA)
+                switch (type) {
+                case AFR_SELF_HEAL_METADATA:
                         nsources = afr_sh_mark_lowest_uid_as_source (bufs,
                                                              success_children,
                                                              child_count,
                                                              sources);
+                        break;
+                case AFR_SELF_HEAL_DATA:
+                        nsources = afr_sh_mark_zero_size_file_as_sink (bufs,
+                                                             success_children,
+                                                             child_count,
+                                                             sources);
+                        if ((nsources < 0) && subvol_status)
+                                *subvol_status |= SPLIT_BRAIN;
+                        break;
+                default:
+                        break;
+                }
                 goto out;
         }
 
@@ -741,7 +1008,8 @@ afr_mark_sources (xlator_t *this, int32_t *sources, int32_t **pending_matrix,
                 nsources = afr_mark_biggest_of_fools_as_source (sources,
                                                                 pending_matrix,
                                                                 characters,
-                                                                child_count);
+								success_children,
+                                                                child_count, bufs);
         }
 
 out:
@@ -759,50 +1027,108 @@ afr_sh_pending_to_delta (afr_private_t *priv, dict_t **xattr,
                          int32_t *delta_matrix[], unsigned char success[],
                          int child_count, afr_transaction_type type)
 {
-        int i = 0;
-        int j = 0;
+        int     tgt     = 0;
+        int     src     = 0;
+        int     value   = 0;
 
         afr_build_pending_matrix (priv->pending_key, delta_matrix, NULL,
                                   xattr, type, priv->child_count);
-        for (i = 0; i < priv->child_count; i++)
-                for (j = 0; j < priv->child_count; j++)
-                        delta_matrix[i][j] = -delta_matrix[i][j];
+
+        /*
+         * The algorithm here has two parts.  First, for each subvol indexed
+         * as tgt, we try to figure out what count everyone should have for it.
+         * If the self-heal succeeded, that's easy; the value is zero.
+         * Otherwise, the value is the maximum of the succeeding nodes' counts.
+         * Once we know the value, we loop through (possibly for a second time)
+         * setting each count to the difference so that when we're done all
+         * succeeding nodes will have the same count for tgt.
+         */
+        for (tgt = 0; tgt < priv->child_count; ++tgt) {
+                value = 0;
+                if (!success[tgt]) {
+                        /* Find the maximum. */
+                        for (src = 0; src < priv->child_count; ++src) {
+                                if (!success[src]) {
+                                        continue;
+                                }
+                                if (delta_matrix[src][tgt] > value) {
+                                        value = delta_matrix[src][tgt];
+                                }
+                        }
+                }
+                /* Force everyone who succeeded to the chosen value. */
+                for (src = 0; src < priv->child_count; ++src) {
+                        if (success[src]) {
+                                delta_matrix[src][tgt] = value
+                                                       - delta_matrix[src][tgt];
+                        }
+                        else {
+                                delta_matrix[src][tgt] = 0;
+                        }
+                }
+        }
 }
 
 
 int
-afr_sh_delta_to_xattr (afr_private_t *priv,
+afr_sh_delta_to_xattr (xlator_t *this,
                        int32_t *delta_matrix[], dict_t *xattr[],
                        int child_count, afr_transaction_type type)
 {
-        int      i       = 0;
-        int      j       = 0;
-        int      k       = 0;
-        int      ret     = 0;
-        int32_t *pending = NULL;
+        int              i       = 0;
+        int              j       = 0;
+        int              k       = 0;
+        int              ret     = 0;
+        int32_t         *pending = NULL;
+        int32_t         *local_pending = NULL;
+        afr_private_t   *priv = NULL;
 
+        priv = this->private;
         for (i = 0; i < child_count; i++) {
                 if (!xattr[i])
                         continue;
 
+                local_pending = NULL;
                 for (j = 0; j < child_count; j++) {
                         pending = GF_CALLOC (sizeof (int32_t), 3,
                                              gf_afr_mt_int32_t);
 
-                        if (!pending)
+                        if (!pending) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "failed to allocate pending entry "
+                                        "for %s[%d] on %s",
+                                        priv->pending_key[j], type,
+                                        priv->children[i]->name);
                                 continue;
+                        }
                         /* 3 = data+metadata+entry */
 
                         k = afr_index_for_transaction_type (type);
 
                         pending[k] = hton32 (delta_matrix[i][j]);
 
+                        if (j == i) {
+                                local_pending = pending;
+                                continue;
+                        }
                         ret = dict_set_bin (xattr[i], priv->pending_key[j],
                                             pending,
-                                            3 * sizeof (int32_t));
-                        if (ret < 0)
-                                gf_log (THIS->name, GF_LOG_WARNING,
+                                        AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_WARNING,
                                         "Unable to set dict value.");
+                                GF_FREE (pending);
+                        }
+                }
+                if (local_pending) {
+                        ret = dict_set_bin (xattr[i], priv->pending_key[i],
+                                            local_pending,
+                                        AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Unable to set dict value.");
+                                GF_FREE (local_pending);
+                        }
                 }
         }
         return 0;
@@ -810,98 +1136,6 @@ afr_sh_delta_to_xattr (afr_private_t *priv,
 
 
 int
-afr_sh_has_metadata_pending (dict_t *xattr, xlator_t *this)
-{
-        /* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-        int32_t        pending[3]  = {0,};
-        void          *pending_raw = NULL;
-        afr_private_t *priv        = NULL;
-        int            ret         = -1;
-        int            i           = 0;
-        int            j           = 0;
-
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_get_ptr (xattr, priv->pending_key[i],
-                                    &pending_raw);
-
-                if (ret != 0)
-                        return 0;
-
-                memcpy (pending, pending_raw, sizeof(pending));
-                j = afr_index_for_transaction_type (AFR_METADATA_TRANSACTION);
-
-                if (pending[j])
-                        return 1;
-        }
-
-        return 0;
-}
-
-
-int
-afr_sh_has_data_pending (dict_t *xattr, xlator_t *this)
-{
-        /* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-        int32_t        pending[3]  = {0,};
-        void          *pending_raw = NULL;
-        afr_private_t *priv        = NULL;
-        int            ret         = -1;
-        int            i           = 0;
-        int            j           = 0;
-
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_get_ptr (xattr, priv->pending_key[i],
-                                    &pending_raw);
-
-                if (ret != 0)
-                        return 0;
-
-                memcpy (pending, pending_raw, sizeof(pending));
-                j = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
-
-                if (pending[j])
-                        return 1;
-        }
-
-        return 0;
-}
-
-
-int
-afr_sh_has_entry_pending (dict_t *xattr, xlator_t *this)
-{
-        /* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-        int32_t        pending[3]  = {0,};
-        void          *pending_raw = NULL;
-        afr_private_t *priv        = NULL;
-        int            ret         = -1;
-        int            i           = 0;
-        int            j           = 0;
-
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_get_ptr (xattr, priv->pending_key[i],
-                                    &pending_raw);
-
-                if (ret != 0)
-                        return 0;
-
-                memcpy (pending, pending_raw, sizeof(pending));
-                j = afr_index_for_transaction_type (AFR_ENTRY_TRANSACTION);
-
-                if (pending[j])
-                        return 1;
-        }
-
-        return 0;
-}
-
-int
 afr_sh_missing_entries_done (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t     *local = NULL;
@@ -911,11 +1145,14 @@ afr_sh_missing_entries_done (call_frame_t *frame, xlator_t *this)
         sh = &local->self_heal;
 
         afr_sh_reset (frame, this);
-        if (local->govinda_gOvinda || sh->op_failed) {
-                gf_log (this->name, GF_LOG_INFO,
+
+        if (local->unhealable) {
+                gf_log (this->name, GF_LOG_DEBUG,
                         "split brain found, aborting selfheal of %s",
                         local->loc.path);
-                sh->op_failed = 1;
+        }
+
+        if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
                 sh->completion_cbk (frame, this);
         } else {
                 gf_log (this->name, GF_LOG_TRACE,
@@ -1001,7 +1238,7 @@ afr_sh_common_lookup_resp_handler (call_frame_t *frame, void *cookie,
                         sh->success_count++;
                         sh->xattr[child_index] = dict_ref (xattr);
                 } else {
-                        gf_log (this->name, GF_LOG_ERROR, "path %s on subvolume"
+                        gf_log (this->name, GF_LOG_DEBUG, "path %s on subvolume"
                                 " %s => -1 (%s)", loc->path,
                                 priv->children[child_index]->name,
                                 strerror (op_errno));
@@ -1147,7 +1384,7 @@ out:
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "impunge of %s failed, "
                         "reason: %s", local->loc.path, strerror (-ret));
-                sh->op_failed = 1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
         }
         afr_sh_missing_entries_finish (frame, this);
 }
@@ -1162,7 +1399,7 @@ afr_sh_create_entry_cbk (call_frame_t *frame, xlator_t *this,
         local = frame->local;
         sh = &local->self_heal;
         if (op_ret < 0)
-                sh->op_failed = 1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
         afr_sh_missing_entries_finish (frame, this);
         return 0;
 }
@@ -1186,7 +1423,7 @@ sh_missing_entries_create (call_frame_t *frame, xlator_t *this)
         if (!afr_valid_ia_type (type)) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "%s: unknown file type: 0%o", local->loc.path, type);
-                local->govinda_gOvinda = 1;
+                afr_set_local_for_unhealable (local);
                 afr_sh_missing_entries_finish (frame, this);
                 goto out;
         }
@@ -1208,6 +1445,10 @@ afr_sh_missing_entries_lookup_done (call_frame_t *frame, xlator_t *this,
         ia_type_t       ia_type = IA_INVAL;
         int32_t         nsources = 0;
         loc_t           *loc = NULL;
+        int32_t         subvol_status = 0;
+        afr_transaction_type txn_type = AFR_DATA_TRANSACTION;
+        gf_boolean_t    split_brain = _gf_false;
+        int             read_child = -1;
 
         local = frame->local;
         sh = &local->self_heal;
@@ -1215,25 +1456,47 @@ afr_sh_missing_entries_lookup_done (call_frame_t *frame, xlator_t *this,
         loc = &local->loc;
 
         if (op_ret < 0) {
-                if (op_errno == EIO)
-                        local->govinda_gOvinda = 1;
+                if (op_errno == EIO) {
+                        afr_set_local_for_unhealable (local);
+                }
                 // EIO can happen if finding the fresh parent dir failed
                 goto out;
         }
 
         //now No chance for the ia_type to conflict
         ia_type = sh->buf[sh->success_children[0]].ia_type;
+        txn_type = afr_transaction_type_get (ia_type);
         nsources = afr_build_sources (this, sh->xattr, sh->buf,
                                       sh->pending_matrix, sh->sources,
-                                      sh->success_children,
-                                      afr_transaction_type_get (ia_type),
-                                      NULL, _gf_false);
+                                      sh->success_children, txn_type,
+                                      &subvol_status, _gf_false);
         if (nsources < 0) {
                 gf_log (this->name, GF_LOG_INFO, "No sources for dir of %s,"
                         " in missing entry self-heal, continuing with the rest"
                         " of the self-heals", local->loc.path);
-                op_errno = EIO;
-                goto out;
+                if (subvol_status & SPLIT_BRAIN) {
+                        split_brain = _gf_true;
+                        switch (txn_type) {
+                        case AFR_DATA_TRANSACTION:
+                                nsources = 1;
+                                sh->sources[sh->success_children[0]] = 1;
+                                break;
+                        case AFR_ENTRY_TRANSACTION:
+                                read_child = afr_get_no_xattr_dir_read_child
+                                                          (this,
+                                                           sh->success_children,
+                                                           sh->buf);
+                                sh->sources[read_child] = 1;
+                                nsources = 1;
+                                break;
+                        default:
+                                op_errno = EIO;
+                                goto out;
+                        }
+                } else {
+                        op_errno = EIO;
+                        goto out;
+                }
         }
 
         afr_get_fresh_children (sh->success_children, sh->sources,
@@ -1250,10 +1513,14 @@ afr_sh_missing_entries_lookup_done (call_frame_t *frame, xlator_t *this,
         sh->type = sh->buf[sh->source].ia_type;
         if (uuid_is_null (loc->inode->gfid))
                 uuid_copy (loc->gfid, sh->buf[sh->source].ia_gfid);
-        sh_missing_entries_create (frame, this);
+        if (split_brain) {
+                afr_sh_missing_entries_finish (frame, this);
+        } else {
+                sh_missing_entries_create (frame, this);
+        }
         return;
 out:
-        sh->op_failed = 1;
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
         afr_sh_set_error (sh, op_errno);
         afr_sh_missing_entries_finish (frame, this);
         return;
@@ -1337,7 +1604,7 @@ afr_sh_remove_entry_cbk (call_frame_t *frame, xlator_t *this, int child,
                 LOCK (&frame->lock);
                 {
                         afr_sh_set_error (sh, EIO);
-                        sh->op_failed = 1;
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 }
                 UNLOCK (&frame->lock);
         }
@@ -1359,6 +1626,7 @@ afr_sh_call_entry_expunge_remove (call_frame_t *frame, xlator_t *this,
         afr_self_heal_t *sh = NULL;
         afr_self_heal_t *expunge_sh = NULL;
         int32_t         op_errno = 0;
+        int             ret = 0;
 
         expunge_frame = copy_frame (frame);
         if (!expunge_frame) {
@@ -1373,6 +1641,12 @@ afr_sh_call_entry_expunge_remove (call_frame_t *frame, xlator_t *this,
         expunge_sh = &expunge_local->self_heal;
         expunge_sh->sh_frame = frame;
         loc_copy (&expunge_local->loc, &local->loc);
+        ret = afr_build_parent_loc (&expunge_sh->parent_loc,
+                                    &expunge_local->loc, &op_errno);
+        if (ret) {
+                ret = -op_errno;
+                goto out;
+        }
         sh->expunge_done = expunge_done;
         afr_sh_entry_expunge_remove (expunge_frame, this, child_index, buf,
                                      parentbuf);
@@ -1412,7 +1686,7 @@ afr_sh_purge_stale_entries_done (call_frame_t *frame, xlator_t *this)
         sh       = &local->self_heal;
         priv     = this->private;
 
-        if (sh->op_failed) {
+        if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
                 afr_sh_missing_entries_finish (frame, this);
         } else {
                 if (afr_gfid_missing_count (this->name, sh->fresh_children,
@@ -1422,7 +1696,8 @@ afr_sh_purge_stale_entries_done (call_frame_t *frame, xlator_t *this)
                                               afr_sh_missing_entries_lookup_done,
                                               sh->sh_gfid_req,
                                               AFR_LOOKUP_FAIL_CONFLICTS|
-                                              AFR_LOOKUP_FAIL_MISSING_GFIDS);
+                                              AFR_LOOKUP_FAIL_MISSING_GFIDS,
+                                              NULL);
                 } else {
                         //No need to set gfid so goto missing entries lookup done
                         //Behave as if you have done the lookup
@@ -1504,7 +1779,7 @@ afr_sh_purge_entry_common (call_frame_t *frame, xlator_t *this,
                 if (!purge_condition (local, priv, i))
                         continue;
                 gf_log (this->name, GF_LOG_INFO, "purging the stale entry %s "
-                        "on %d", local->loc.path, i);
+                        "on %s", local->loc.path, priv->children[i]->name);
                 afr_sh_call_entry_expunge_remove (frame, this,
                                                   (long) i, &sh->buf[i],
                                                   &sh->parentbufs[i],
@@ -1624,10 +1899,8 @@ afr_sh_children_lookup_done (call_frame_t *frame, xlator_t *this,
                                                sh->child_errno,
                                                priv->child_count, ENOENT);
         if (fresh_child_enoents == fresh_parent_count) {
-                gf_log (this->name, GF_LOG_INFO, "Deleting stale file %s",
-                        local->loc.path);
                 afr_sh_set_error (sh, ENOENT);
-                sh->op_failed = 1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 afr_sh_purge_entry (frame, this);
         } else if (!afr_conflicting_iattrs (sh->buf, sh->fresh_children,
                                             priv->child_count, local->loc.path,
@@ -1641,14 +1914,14 @@ afr_sh_children_lookup_done (call_frame_t *frame, xlator_t *this,
                 afr_sh_purge_stale_entry (frame, this);
         } else {
                 op_errno = EIO;
-                local->govinda_gOvinda = 1;
+                afr_set_local_for_unhealable (local);
                 goto fail;
         }
 
         return;
 
 fail:
-        sh->op_failed = 1;
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
         afr_sh_set_error (sh, op_errno);
         afr_sh_missing_entries_finish (frame, this);
         return;
@@ -1699,6 +1972,7 @@ afr_sh_find_fresh_parents (call_frame_t *frame, xlator_t *this,
                 gf_log (this->name, GF_LOG_ERROR, "No sources for dir "
                         "of %s, in missing entry self-heal, aborting "
                         "self-heal", local->loc.path);
+                op_errno = EIO;
                 goto out;
         }
 
@@ -1706,18 +1980,20 @@ afr_sh_find_fresh_parents (call_frame_t *frame, xlator_t *this,
         if (source == -1) {
                 GF_ASSERT (0);
                 gf_log (this->name, GF_LOG_DEBUG, "No active sources found.");
+                op_errno = EIO;
                 goto out;
         }
         afr_get_fresh_children (sh->success_children, sh->sources,
                                 sh->fresh_parent_dirs, priv->child_count);
         afr_sh_common_lookup (frame, this, &local->loc,
-                              afr_sh_children_lookup_done, NULL, 0);
+                              afr_sh_children_lookup_done, NULL, 0,
+                              NULL);
         return;
 
 out:
-        afr_sh_set_error (sh, EIO);
-        sh->op_failed = 1;
-        afr_sh_missing_entries_finish (frame, this);
+        afr_sh_set_error (sh, op_errno);
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+	afr_sh_missing_entries_finish (frame, this);
         return;
 }
 
@@ -1745,7 +2021,7 @@ afr_sh_common_reset (afr_self_heal_t *sh, unsigned int child_count)
 int
 afr_sh_common_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
                       afr_lookup_done_cbk_t lookup_done , uuid_t gfid,
-                      int32_t flags)
+                      int32_t flags, dict_t *xdata)
 {
         afr_local_t    *local = NULL;
         int             i = 0;
@@ -1806,7 +2082,8 @@ afr_sh_common_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 
 int
-afr_sh_post_nb_entrylk_conflicting_sh_cbk (call_frame_t *frame, xlator_t *this)
+afr_sh_post_nb_entrylk_missing_entry_sh_cbk (call_frame_t *frame,
+                                             xlator_t *this)
 {
         afr_internal_lock_t *int_lock = NULL;
         afr_local_t         *local    = NULL;
@@ -1819,7 +2096,7 @@ afr_sh_post_nb_entrylk_conflicting_sh_cbk (call_frame_t *frame, xlator_t *this)
         if (int_lock->lock_op_ret < 0) {
                 gf_log (this->name, GF_LOG_INFO,
                         "Non blocking entrylks failed.");
-                sh->op_failed = -1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 afr_sh_missing_entries_done (frame, this);
         } else {
 
@@ -1827,34 +2104,8 @@ afr_sh_post_nb_entrylk_conflicting_sh_cbk (call_frame_t *frame, xlator_t *this)
                         "Non blocking entrylks done. Proceeding to FOP");
                 afr_sh_common_lookup (frame, this, &sh->parent_loc,
                                       afr_sh_find_fresh_parents,
-                                      NULL, AFR_LOOKUP_FAIL_CONFLICTS);
-        }
-
-        return 0;
-}
-
-int
-afr_sh_post_nb_entrylk_gfid_sh_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
-
-        local    = frame->local;
-        sh       = &local->self_heal;
-        int_lock = &local->internal_lock;
-
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Non blocking entrylks failed.");
-                afr_sh_missing_entries_done (frame, this);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non blocking entrylks done. Proceeding to FOP");
-                afr_sh_common_lookup (frame, this, &local->loc,
-                                      afr_sh_missing_entries_lookup_done,
-                                      sh->sh_gfid_req, AFR_LOOKUP_FAIL_CONFLICTS|
-                                      AFR_LOOKUP_FAIL_MISSING_GFIDS);
+                                      NULL, AFR_LOOKUP_FAIL_CONFLICTS,
+                                      NULL);
         }
 
         return 0;
@@ -1866,7 +2117,9 @@ afr_sh_entrylk (call_frame_t *frame, xlator_t *this, loc_t *loc,
 {
         afr_internal_lock_t *int_lock = NULL;
         afr_local_t         *local    = NULL;
+        afr_private_t       *priv     = NULL;
 
+        priv     = this->private;
         local    = frame->local;
         int_lock = &local->internal_lock;
 
@@ -1878,7 +2131,12 @@ afr_sh_entrylk (call_frame_t *frame, xlator_t *this, loc_t *loc,
         int_lock->lk_basename = base_name;
         int_lock->lk_loc      = loc;
         int_lock->lock_cbk    = lock_cbk;
+        int_lock->domain      = this->name;
 
+        int_lock->lockee_count = 0;
+        afr_init_entry_lockee (&int_lock->lockee[0], local, loc,
+                               base_name, priv->child_count);
+        int_lock->lockee_count++;
         afr_nonblocking_entrylk (frame, this);
 
         return 0;
@@ -1916,27 +2174,31 @@ out:
 }
 
 static int
-afr_self_heal_conflicting_entries (call_frame_t *frame, xlator_t *this)
+afr_self_heal_missing_entries (call_frame_t *frame, xlator_t *this)
 {
-        afr_self_heal_parent_entrylk (frame, this,
-                                      afr_sh_post_nb_entrylk_conflicting_sh_cbk);
-        return 0;
-}
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+
+        local = frame->local;
+        sh = &local->self_heal;
+
+        sh->sh_type_in_action  = AFR_SELF_HEAL_GFID_OR_MISSING_ENTRY;
+
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_STARTED);
 
-static int
-afr_self_heal_gfids (call_frame_t *frame, xlator_t *this)
-{
         afr_self_heal_parent_entrylk (frame, this,
-                                      afr_sh_post_nb_entrylk_gfid_sh_cbk);
+                                      afr_sh_post_nb_entrylk_missing_entry_sh_cbk);
         return 0;
 }
 
-afr_local_t *afr_local_copy (afr_local_t *l, xlator_t *this)
+afr_local_t*
+afr_self_heal_local_init (afr_local_t *l, xlator_t *this)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t   *lc     = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_self_heal_t *shc = NULL;
+        afr_private_t   *priv  = NULL;
+        afr_local_t     *lc    = NULL;
+        afr_self_heal_t *sh    = NULL;
+        afr_self_heal_t *shc   = NULL;
+        int             ret    = 0;
 
         priv = this->private;
 
@@ -1955,16 +2217,27 @@ afr_local_t *afr_local_copy (afr_local_t *l, xlator_t *this)
         shc->do_data_self_heal = sh->do_data_self_heal;
         shc->do_metadata_self_heal = sh->do_metadata_self_heal;
         shc->do_entry_self_heal = sh->do_entry_self_heal;
+        shc->force_confirm_spb = sh->force_confirm_spb;
         shc->forced_merge = sh->forced_merge;
         shc->background = sh->background;
         shc->type = sh->type;
+        shc->data_sh_info = "";
+        shc->metadata_sh_info =  "";
 
         uuid_copy (shc->sh_gfid_req, sh->sh_gfid_req);
-        if (l->loc.path)
-                loc_copy (&lc->loc, &l->loc);
+        if (l->loc.path) {
+                ret = loc_copy (&lc->loc, &l->loc);
+                if (ret < 0)
+                        goto out;
+        }
 
         lc->child_up  = memdup (l->child_up,
                                 sizeof (*lc->child_up) * priv->child_count);
+        if (!lc->child_up) {
+                ret = -1;
+                goto out;
+        }
+
         if (l->xattr_req)
                 lc->xattr_req = dict_ref (l->xattr_req);
 
@@ -1972,40 +2245,25 @@ afr_local_t *afr_local_copy (afr_local_t *l, xlator_t *this)
                 lc->cont.lookup.inode = inode_ref (l->cont.lookup.inode);
         if (l->cont.lookup.xattr)
                 lc->cont.lookup.xattr = dict_ref (l->cont.lookup.xattr);
-        if (l->internal_lock.inode_locked_nodes)
-                lc->internal_lock.inode_locked_nodes =
-                        memdup (l->internal_lock.inode_locked_nodes,
-                                sizeof (*lc->internal_lock.inode_locked_nodes) * priv->child_count);
-        else
-                lc->internal_lock.inode_locked_nodes =
-                        GF_CALLOC (sizeof (*l->internal_lock.inode_locked_nodes),
-                                   priv->child_count,
-                                   gf_afr_mt_char);
-        if (l->internal_lock.entry_locked_nodes)
-                lc->internal_lock.entry_locked_nodes =
-                        memdup (l->internal_lock.entry_locked_nodes,
-                                sizeof (*lc->internal_lock.entry_locked_nodes) * priv->child_count);
-        else
-                lc->internal_lock.entry_locked_nodes =
-                        GF_CALLOC (sizeof (*l->internal_lock.entry_locked_nodes),
-                                   priv->child_count,
-                                   gf_afr_mt_char);
-        if (l->internal_lock.locked_nodes)
-                lc->internal_lock.locked_nodes =
-                        memdup (l->internal_lock.locked_nodes,
-                                sizeof (*lc->internal_lock.locked_nodes) * priv->child_count);
-        else
-                lc->internal_lock.locked_nodes =
-                        GF_CALLOC (sizeof (*l->internal_lock.locked_nodes),
-                                   priv->child_count,
-                                   gf_afr_mt_char);
 
-        lc->internal_lock.inodelk_lock_count =
-                l->internal_lock.inodelk_lock_count;
-        lc->internal_lock.entrylk_lock_count =
-                l->internal_lock.entrylk_lock_count;
+        lc->internal_lock.locked_nodes =
+                             GF_CALLOC (sizeof (*l->internal_lock.locked_nodes),
+                                        priv->child_count, gf_afr_mt_char);
+        if (!lc->internal_lock.locked_nodes) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = afr_inodelk_init (&lc->internal_lock.inodelk[0],
+                                this->name, priv->child_count);
+        if (ret)
+                goto out;
 
 out:
+        if (ret) {
+                afr_local_cleanup (lc, this);
+                lc = NULL;
+        }
         return lc;
 }
 
@@ -2015,32 +2273,39 @@ afr_self_heal_completion_cbk (call_frame_t *bgsh_frame, xlator_t *this)
         afr_private_t *   priv  = NULL;
         afr_local_t *     local = NULL;
         afr_self_heal_t * sh    = NULL;
+        afr_local_t *     orig_frame_local = NULL;
+        afr_self_heal_t * orig_frame_sh = NULL;
         char              sh_type_str[256] = {0,};
-        gf_boolean_t      split_brain = _gf_false;
+        gf_loglevel_t     loglevel = 0;
 
         priv  = this->private;
         local = bgsh_frame->local;
         sh    = &local->self_heal;
 
-        if (local->govinda_gOvinda)
-                split_brain = _gf_true;
-
-        afr_set_split_brain (this, sh->inode, split_brain);
+        if (local->unhealable) {
+                afr_set_split_brain (this, sh->inode, SPB, SPB);
+        }
 
         afr_self_heal_type_str_get (sh, sh_type_str,
                                     sizeof(sh_type_str));
-        if (sh->op_failed) {
-                gf_log (this->name, GF_LOG_ERROR, "background %s self-heal "
-                        "failed on %s", sh_type_str, local->loc.path);
+        if (is_self_heal_failed (sh, AFR_CHECK_ALL) && !priv->shd.iamshd) {
+                loglevel = GF_LOG_ERROR;
+        } else if (!is_self_heal_failed (sh, AFR_CHECK_ALL)) {
+                loglevel = GF_LOG_INFO;
         } else {
-                gf_log (this->name, GF_LOG_INFO, "background %s self-heal "
-                        "completed on %s", sh_type_str, local->loc.path);
+                loglevel = GF_LOG_DEBUG;
         }
 
+        afr_log_self_heal_completion_status (local, loglevel);
+
         FRAME_SU_UNDO (bgsh_frame, afr_local_t);
 
         if (!sh->unwound && sh->unwind) {
-                sh->unwind (sh->orig_frame, this, sh->op_ret, sh->op_errno);
+                orig_frame_local = sh->orig_frame->local;
+                orig_frame_sh = &orig_frame_local->self_heal;
+                orig_frame_sh->actual_sh_started = _gf_true;
+                sh->unwind (sh->orig_frame, this, sh->op_ret, sh->op_errno,
+                            is_self_heal_failed (sh, AFR_CHECK_ALL));
         }
 
         if (sh->background) {
@@ -2089,14 +2354,13 @@ afr_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode)
         afr_set_lk_owner (sh_frame, this, sh_frame->root);
         afr_set_low_priority (sh_frame);
 
-        sh_local        = afr_local_copy (local, this);
+        sh_local        = afr_self_heal_local_init (local, this);
         if (!sh_local)
                 goto out;
         sh_frame->local = sh_local;
         sh              = &sh_local->self_heal;
 
         sh->inode       = inode_ref (inode);
-
         sh->orig_frame  = frame;
 
         sh->completion_cbk = afr_self_heal_completion_cbk;
@@ -2155,12 +2419,11 @@ afr_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode)
                 sh->do_gfid_self_heal = _gf_false;
         }
 
+        sh->sh_type_in_action = AFR_SELF_HEAL_INVALID;
+
         FRAME_SU_DO (sh_frame, afr_local_t);
-        if (sh->do_missing_entry_self_heal) {
-                afr_self_heal_conflicting_entries (sh_frame, this);
-        } else if (sh->do_gfid_self_heal) {
-                GF_ASSERT (!uuid_is_null (sh->sh_gfid_req));
-                afr_self_heal_gfids (sh_frame, this);
+        if (sh->do_missing_entry_self_heal || sh->do_gfid_self_heal) {
+                afr_self_heal_missing_entries (sh_frame, this);
         } else {
                 loc = &sh_local->loc;
                 if (uuid_is_null (loc->inode->gfid) && uuid_is_null (loc->gfid)) {
@@ -2179,7 +2442,7 @@ afr_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode)
 
 out:
         if (op_errno) {
-                orig_sh->unwind (frame, this, -1, op_errno);
+                orig_sh->unwind (frame, this, -1, op_errno, 1);
                 if (sh_frame)
                         AFR_STACK_DESTROY (sh_frame);
         }
@@ -2285,3 +2548,265 @@ out:
 
         return ret;
 }
+
+int
+afr_sh_erase_pending (call_frame_t *frame, xlator_t *this,
+                      afr_transaction_type type, afr_fxattrop_cbk_t cbk,
+                      int (*finish)(call_frame_t *frame, xlator_t *this))
+{
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+        int              call_count = 0;
+        int              i = 0;
+        dict_t          **erase_xattr = NULL;
+        int             ret = -1;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        afr_sh_pending_to_delta (priv, sh->xattr, sh->delta_matrix,
+                                 sh->success, priv->child_count, type);
+
+        erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count,
+                                 gf_afr_mt_dict_t);
+        if (!erase_xattr)
+                goto out;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (sh->xattr[i]) {
+                        call_count++;
+                        erase_xattr[i] = dict_new ();
+                        if (!erase_xattr[i])
+                                goto out;
+                }
+        }
+
+        afr_sh_delta_to_xattr (this, sh->delta_matrix, erase_xattr,
+                               priv->child_count, type);
+
+        gf_log (this->name, GF_LOG_DEBUG, "Delta matrix for: %s",
+                lkowner_utoa (&frame->root->lk_owner));
+        afr_sh_print_pending_matrix (sh->delta_matrix, this);
+        local->call_count = call_count;
+        if (call_count == 0) {
+                ret = 0;
+                finish (frame, this);
+                goto out;
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (!erase_xattr[i])
+                        continue;
+
+                if (sh->healing_fd) {//true for ENTRY, reg file DATA transaction
+                        STACK_WIND_COOKIE (frame, cbk, (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->fxattrop,
+                                           sh->healing_fd,
+                                           GF_XATTROP_ADD_ARRAY, erase_xattr[i],
+                                           NULL);
+                } else {
+                        STACK_WIND_COOKIE (frame, cbk, (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->xattrop,
+                                           &local->loc,
+                                           GF_XATTROP_ADD_ARRAY, erase_xattr[i],
+                                           NULL);
+                }
+        }
+
+        ret = 0;
+out:
+        if (erase_xattr) {
+                for (i = 0; i < priv->child_count; i++) {
+                        if (erase_xattr[i]) {
+                                dict_unref (erase_xattr[i]);
+                        }
+                }
+        }
+
+        GF_FREE (erase_xattr);
+
+        if (ret < 0) {
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                finish (frame, this);
+        }
+
+        return 0;
+}
+
+void
+afr_set_self_heal_status(afr_self_heal_t *sh, afr_self_heal_status status)
+{
+        xlator_t                *this = NULL;
+        afr_sh_status_for_all_type *sh_status = &(sh->afr_all_sh_status);
+        afr_self_heal_type  sh_type_in_action = sh->sh_type_in_action;
+        this = THIS;
+
+        if (!sh) {
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "Null self heal"
+                                  "Structure");
+                goto out;
+        }
+
+        switch (sh_type_in_action) {
+                case AFR_SELF_HEAL_GFID_OR_MISSING_ENTRY:
+                       sh_status->gfid_or_missing_entry_self_heal = status;
+                        break;
+                case AFR_SELF_HEAL_METADATA:
+                        sh_status->metadata_self_heal = status;
+                        break;
+                case AFR_SELF_HEAL_DATA:
+                        sh_status->data_self_heal = status;
+                        break;
+                case AFR_SELF_HEAL_ENTRY:
+                        sh_status->entry_self_heal = status;
+                        break;
+                case AFR_SELF_HEAL_INVALID:
+                        gf_log_callingfn (this->name, GF_LOG_ERROR, "Invalid"
+                                          "self heal type in action");
+                        break;
+        }
+out:
+        return;
+}
+
+void
+afr_set_local_for_unhealable (afr_local_t *local)
+{
+        afr_self_heal_t  *sh = NULL;
+
+        sh = &local->self_heal;
+
+        local->unhealable = 1;
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+}
+
+int
+is_self_heal_failed (afr_self_heal_t *sh, afr_sh_fail_check_type type)
+{
+        afr_sh_status_for_all_type      sh_status = sh->afr_all_sh_status;
+        afr_self_heal_type   sh_type_in_action =  AFR_SELF_HEAL_INVALID;
+        afr_self_heal_status    status = AFR_SELF_HEAL_FAILED;
+        xlator_t                *this = NULL;
+        int                     sh_failed = 0;
+
+        this = THIS;
+
+        if (!sh) {
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "Null self heal "
+                                  "structure");
+                sh_failed = 1;
+                goto out;
+        }
+
+        if (type == AFR_CHECK_ALL) {
+                if ((sh_status.gfid_or_missing_entry_self_heal == AFR_SELF_HEAL_FAILED)
+                    || (sh_status.metadata_self_heal == AFR_SELF_HEAL_FAILED)
+                    || (sh_status.data_self_heal == AFR_SELF_HEAL_FAILED)
+                    || (sh_status.entry_self_heal == AFR_SELF_HEAL_FAILED))
+                sh_failed = 1;
+        } else if (type == AFR_CHECK_SPECIFIC) {
+                sh_type_in_action = sh->sh_type_in_action;
+                switch (sh_type_in_action) {
+                        case AFR_SELF_HEAL_GFID_OR_MISSING_ENTRY:
+                             status = sh_status.gfid_or_missing_entry_self_heal;
+                                break;
+                        case AFR_SELF_HEAL_METADATA:
+                                status = sh_status.metadata_self_heal;
+                                break;
+                        case AFR_SELF_HEAL_ENTRY:
+                                status = sh_status.entry_self_heal;
+                                break;
+                        case AFR_SELF_HEAL_DATA:
+                                status = sh_status.data_self_heal;
+                                break;
+                        case AFR_SELF_HEAL_INVALID:
+                                status = AFR_SELF_HEAL_NOT_ATTEMPTED;
+                                break;
+                }
+                if (status == AFR_SELF_HEAL_FAILED)
+                        sh_failed = 1;
+
+        }
+
+out:
+        return sh_failed;
+}
+
+char *
+get_sh_completion_status (afr_self_heal_status status)
+{
+
+        char *not_attempted       = " is not attempted";
+        char *failed              = " failed";
+        char *started             = " is started";
+        char *sync_begin          = " is successfully completed";
+        char *result              = " has unknown status";
+
+        switch (status)
+        {
+                case AFR_SELF_HEAL_NOT_ATTEMPTED:
+                        result = not_attempted;
+                        break;
+                case AFR_SELF_HEAL_FAILED:
+                        result = failed;
+                        break;
+                case AFR_SELF_HEAL_STARTED:
+                        result = started;
+                        break;
+                case AFR_SELF_HEAL_SYNC_BEGIN:
+                        result = sync_begin;
+                        break;
+        }
+
+        return result;
+
+}
+
+void
+afr_log_self_heal_completion_status (afr_local_t *local, gf_loglevel_t loglvl)
+{
+
+        char sh_log[4096]              = {0};
+        afr_self_heal_t *sh            = &local->self_heal;
+        afr_sh_status_for_all_type   all_status = sh->afr_all_sh_status;
+        xlator_t      *this            = NULL;
+        size_t        off              = 0;
+        int           data_sh          = 0;
+        int           metadata_sh      = 0;
+        int           print_log        = 0;
+
+        this = THIS;
+
+        ADD_FMT_STRING (sh_log, off, "gfid or missing entry",
+                        all_status.gfid_or_missing_entry_self_heal, print_log);
+        ADD_FMT_STRING_SYNC (sh_log, off, "metadata",
+                             all_status.metadata_self_heal, print_log);
+        if (sh->background) {
+                ADD_FMT_STRING_SYNC (sh_log, off, "backgroung data",
+                                all_status.data_self_heal, print_log);
+        } else {
+                ADD_FMT_STRING_SYNC (sh_log, off, "foreground data",
+                                all_status.data_self_heal, print_log);
+        }
+        ADD_FMT_STRING_SYNC (sh_log, off, "entry", all_status.entry_self_heal,
+                             print_log);
+
+        if (AFR_SELF_HEAL_SYNC_BEGIN == all_status.data_self_heal &&
+	    strcmp (sh->data_sh_info, "") && sh->data_sh_info )
+                data_sh = 1;
+        if (AFR_SELF_HEAL_SYNC_BEGIN == all_status.metadata_self_heal &&
+	    strcmp (sh->metadata_sh_info, "") && sh->metadata_sh_info)
+                metadata_sh = 1;
+
+        if (!print_log)
+                return;
+
+        gf_log (this->name, loglvl, "%s %s %s on %s", sh_log,
+                ((data_sh == 1) ? sh->data_sh_info : ""),
+                ((metadata_sh == 1) ? sh->metadata_sh_info : ""),
+                local->loc.path);
+}
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.h b/xlators/cluster/afr/src/afr-self-heal-common.h
index 24cac6232..473264776 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.h
+++ b/xlators/cluster/afr/src/afr-self-heal-common.h
@@ -1,33 +1,18 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __AFR_SELF_HEAL_COMMON_H__
 #define __AFR_SELF_HEAL_COMMON_H__
 
 #define FILE_HAS_HOLES(buf) (((buf)->ia_size) > ((buf)->ia_blocks * 512))
-
-typedef enum {
-        AFR_SELF_HEAL_ENTRY,
-        AFR_SELF_HEAL_METADATA,
-        AFR_SELF_HEAL_DATA,
-        AFR_SELF_HEAL_INVALID = -1,
-} afr_self_heal_type;
+#define AFR_SH_MIN_PARTICIPANTS 2
 
 typedef enum {
         AFR_LOOKUP_FAIL_CONFLICTS = 1,
@@ -43,6 +28,10 @@ afr_sh_source_count (int sources[], int child_count);
 void
 afr_sh_print_pending_matrix (int32_t *pending_matrix[], xlator_t *this);
 
+void
+afr_sh_print_split_brain_log (int32_t *pending_matrix[], xlator_t *this,
+                              const char *loc);
+
 int
 afr_build_pending_matrix (char **pending_key, int32_t **pending_matrix,
                           unsigned char *ignorant_subvols,
@@ -60,7 +49,7 @@ afr_mark_sources (xlator_t *this, int32_t *sources, int32_t **pending_matrix,
                   int32_t *success_children, int32_t *subvol_status);
 
 int
-afr_sh_delta_to_xattr (afr_private_t *priv,
+afr_sh_delta_to_xattr (xlator_t *this,
                        int32_t *delta_matrix[], dict_t *xattr[],
 		       int child_count, afr_transaction_type type);
 
@@ -90,7 +79,7 @@ afr_sh_common_lookup_resp_handler (call_frame_t *frame, void *cookie,
 int
 afr_sh_common_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
                       afr_lookup_done_cbk_t lookup_cbk, uuid_t uuid,
-                      int32_t flags);
+                      int32_t flags, dict_t *xdata);
 int
 afr_sh_entry_expunge_remove (call_frame_t *expunge_frame, xlator_t *this,
                              int active_src, struct iatt *buf,
@@ -102,13 +91,13 @@ int
 afr_sh_entry_impunge_create (call_frame_t *impunge_frame, xlator_t *this,
                              int child_index);
 int
-afr_sh_data_unlock (call_frame_t *frame, xlator_t *this,
+afr_sh_data_unlock (call_frame_t *frame, xlator_t *this, char *dom,
                     afr_lock_cbk_t lock_cbk);
 afr_local_t *
-afr_local_copy (afr_local_t *l, xlator_t *this);
+afr_self_heal_local_init (afr_local_t *l, xlator_t *this);
 int
 afr_sh_data_lock (call_frame_t *frame, xlator_t *this,
-                  off_t start, off_t len,
+                  off_t start, off_t len, gf_boolean_t block, char *dom,
                   afr_lock_cbk_t success_handler,
                   afr_lock_cbk_t failure_handler);
 void
@@ -118,7 +107,7 @@ afr_sh_mark_source_sinks (call_frame_t *frame, xlator_t *this);
 typedef int
 (*afr_fxattrop_cbk_t) (call_frame_t *frame, void *cookie,
                        xlator_t *this, int32_t op_ret, int32_t op_errno,
-                       dict_t *xattr);
+                       dict_t *xattr, dict_t *xdata);
 int
 afr_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name);
 int
@@ -130,4 +119,26 @@ afr_sh_reset (call_frame_t *frame, xlator_t *this);
 void
 afr_children_intersection_get (int32_t *set1, int32_t *set2,
                                int *intersection, unsigned int child_count);
+int
+afr_get_no_xattr_dir_read_child (xlator_t *this, int32_t *success_children,
+                                 struct iatt *bufs);
+int
+afr_sh_erase_pending (call_frame_t *frame, xlator_t *this,
+                      afr_transaction_type type, afr_fxattrop_cbk_t cbk,
+                      int (*finish)(call_frame_t *frame, xlator_t *this));
+
+void
+afr_set_local_for_unhealable (afr_local_t *local);
+
+int
+is_self_heal_failed (afr_self_heal_t *sh, afr_sh_fail_check_type type);
+
+void
+afr_set_self_heal_status (afr_self_heal_t *sh, afr_self_heal_status status);
+
+void
+afr_log_self_heal_completion_status (afr_local_t *local, gf_loglevel_t  logl);
+
+char*
+afr_get_pending_matrix_str (int32_t *pending_matrix[], xlator_t *this);
 #endif /* __AFR_SELF_HEAL_COMMON_H__ */
diff --git a/xlators/cluster/afr/src/afr-self-heal-data.c b/xlators/cluster/afr/src/afr-self-heal-data.c
index 765edd277..9de26ee56 100644
--- a/xlators/cluster/afr/src/afr-self-heal-data.c
+++ b/xlators/cluster/afr/src/afr-self-heal-data.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -49,6 +40,14 @@
 #include "afr-self-heal-common.h"
 #include "afr-self-heal-algorithm.h"
 
+int
+afr_sh_data_fail (call_frame_t *frame, xlator_t *this);
+
+static inline gf_boolean_t
+afr_sh_data_proceed (unsigned int success_count)
+{
+        return (success_count >= AFR_SH_MIN_PARTICIPANTS);
+}
 
 extern int
 sh_loop_finish (call_frame_t *loop_frame, xlator_t *this);
@@ -63,15 +62,6 @@ int
 afr_sh_data_finish (call_frame_t *frame, xlator_t *this);
 
 int
-afr_sh_data_fxattrop (call_frame_t *frame, xlator_t *this,
-                      afr_fxattrop_cbk_t fxattrop_cbk);
-
-int
-afr_post_sh_data_fxattrop_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret, int32_t op_errno,
-                               dict_t *xattr);
-
-int
 afr_sh_data_done (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t     *local = NULL;
@@ -88,7 +78,7 @@ afr_sh_data_done (call_frame_t *frame, xlator_t *this)
 
 int
 afr_sh_data_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t   *local       = NULL;
         afr_private_t *priv        = NULL;
@@ -101,7 +91,7 @@ afr_sh_data_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         LOCK (&frame->lock);
         {
                 if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_INFO,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "flush failed on %s on subvolume %s: %s",
                                 local->loc.path, priv->children[child_index]->name,
                                 strerror (op_errno));
@@ -131,6 +121,11 @@ afr_sh_data_close (call_frame_t *frame, xlator_t *this)
         sh    = &local->self_heal;
         priv  = this->private;
 
+        if (!sh->healing_fd) {
+                //This happens when file is non-reg
+                afr_sh_data_done (frame, this);
+                return 0;
+        }
         call_count        = afr_set_elem_count_get (sh->success,
                                                     priv->child_count);
         local->call_count = call_count;
@@ -151,7 +146,7 @@ afr_sh_data_close (call_frame_t *frame, xlator_t *this)
                                    (void *) (long) i,
                                    priv->children[i],
                                    priv->children[i]->fops->flush,
-                                   sh->healing_fd);
+                                   sh->healing_fd, NULL);
 
                 if (!--call_count)
                         break;
@@ -161,9 +156,28 @@ afr_sh_data_close (call_frame_t *frame, xlator_t *this)
 }
 
 int
+afr_sh_dom_unlock (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh    = NULL;
+        afr_private_t   *priv  = NULL;
+
+        local = frame->local;
+        sh    = &local->self_heal;
+        priv  = this->private;
+
+        if (sh->sh_dom_lock_held)
+                afr_sh_data_unlock (frame, this, priv->sh_domain,
+                                    afr_sh_data_close);
+        else
+                afr_sh_data_close (frame, this);
+        return 0;
+}
+
+int
 afr_sh_data_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                         struct iatt *statpost)
+                         struct iatt *statpost, dict_t *xdata)
 {
 
         afr_local_t   *local       = NULL;
@@ -195,29 +209,20 @@ afr_sh_data_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 }
 
 int
-afr_sh_data_setattr (call_frame_t *frame, xlator_t *this)
+afr_sh_data_setattr (call_frame_t *frame, xlator_t *this, struct iatt* stbuf)
 {
         afr_local_t     *local      = NULL;
         afr_private_t   *priv       = NULL;
         afr_self_heal_t *sh         = NULL;
         int              i          = 0;
         int              call_count = 0;
-        int              source     = 0;
         int32_t          valid      = 0;
-        struct iatt      stbuf      = {0,};
 
         local = frame->local;
         sh    = &local->self_heal;
         priv  = this->private;
 
-        source = sh->source;
-
-        valid |= (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME);
-
-        stbuf.ia_atime = sh->buf[source].ia_atime;
-        stbuf.ia_atime_nsec = sh->buf[source].ia_atime_nsec;
-        stbuf.ia_mtime = sh->buf[source].ia_mtime;
-        stbuf.ia_mtime_nsec = sh->buf[source].ia_mtime_nsec;
+        valid = (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME);
 
         call_count        = afr_set_elem_count_get (sh->success,
                                                     priv->child_count);
@@ -237,7 +242,7 @@ afr_sh_data_setattr (call_frame_t *frame, xlator_t *this)
                                    (void *) (long) i,
                                    priv->children[i],
                                    priv->children[i]->fops->setattr,
-                                   &local->loc, &stbuf, valid);
+                                   &local->loc, stbuf, valid, NULL);
 
                 if (!--call_count)
                         break;
@@ -249,7 +254,7 @@ afr_sh_data_setattr (call_frame_t *frame, xlator_t *this)
 int
 afr_sh_data_setattr_fstat_cbk (call_frame_t *frame, void *cookie,
                                xlator_t *this, int32_t op_ret, int32_t op_errno,
-                               struct iatt *buf)
+                               struct iatt *buf, dict_t *xdata)
 {
         afr_local_t     *local = NULL;
         afr_self_heal_t *sh = NULL;
@@ -259,9 +264,14 @@ afr_sh_data_setattr_fstat_cbk (call_frame_t *frame, void *cookie,
         sh = &local->self_heal;
 
         GF_ASSERT (sh->source == child_index);
-        if (op_ret != -1)
+        if (op_ret != -1) {
                 sh->buf[child_index] = *buf;
-        afr_sh_data_setattr (frame, this);
+                afr_sh_data_setattr (frame, this, buf);
+        } else {
+                gf_log (this->name, GF_LOG_ERROR, "%s: Failed to set "
+                        "time-stamps after self-heal", local->loc.path);
+                afr_sh_data_fail (frame, this);
+        }
 
         return 0;
 }
@@ -286,37 +296,51 @@ afr_sh_set_timestamps (call_frame_t *frame, xlator_t *this)
                            (void *) (long) sh->source,
                            priv->children[sh->source],
                            priv->children[sh->source]->fops->fstat,
-                           sh->healing_fd);
+                           sh->healing_fd, NULL);
         return 0;
 }
 
 //Fun fact, lock_cbk is being used for both lock & unlock
 int
-afr_sh_data_unlock (call_frame_t *frame, xlator_t *this,
+afr_sh_data_unlock (call_frame_t *frame, xlator_t *this, char *dom,
                     afr_lock_cbk_t lock_cbk)
 {
         afr_local_t         *local    = NULL;
         afr_internal_lock_t *int_lock = NULL;
         afr_self_heal_t     *sh       = NULL;
+        afr_private_t       *priv     = NULL;
+        int                 ret       = 0;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
         sh       = &local->self_heal;
+        priv     = this->private;
 
-        GF_ASSERT (sh->data_lock_held);
-
-        sh->data_lock_held = _gf_false;
+        if (strcmp (dom, this->name) == 0) {
+                sh->data_lock_held = _gf_false;
+        } else if (strcmp (dom, priv->sh_domain) == 0) {
+                sh->sh_dom_lock_held = _gf_false;
+        } else {
+                ret = -1;
+                goto out;
+        }
         int_lock->lock_cbk = lock_cbk;
+        int_lock->domain = dom;
         afr_unlock (frame, this);
 
+out:
+        if (ret) {
+                int_lock->lock_op_ret = -1;
+                int_lock->lock_cbk (frame, this);
+        }
         return 0;
 }
 
 int
 afr_sh_data_finish (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t   *local = NULL;
-        afr_self_heal_t *sh = NULL;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh    = NULL;
 
         local = frame->local;
         sh = &local->self_heal;
@@ -325,9 +349,9 @@ afr_sh_data_finish (call_frame_t *frame, xlator_t *this)
                 "finishing data selfheal of %s", local->loc.path);
 
         if (sh->data_lock_held)
-                afr_sh_data_unlock (frame, this, afr_sh_data_close);
+                afr_sh_data_unlock (frame, this, this->name, afr_sh_dom_unlock);
         else
-                afr_sh_data_close (frame, this);
+                afr_sh_dom_unlock (frame, this);
 
         return 0;
 }
@@ -344,34 +368,49 @@ afr_sh_data_fail (call_frame_t *frame, xlator_t *this)
         gf_log (this->name, GF_LOG_DEBUG,
                 "finishing failed data selfheal of %s", local->loc.path);
 
-        sh->op_failed = 1;
-        if (sh->data_lock_held)
-                afr_sh_data_unlock (frame, this, afr_sh_data_close);
-        else
-                afr_sh_data_close (frame, this);
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+        afr_sh_data_finish (frame, this);
         return 0;
 }
 
 int
 afr_sh_data_erase_pending_cbk (call_frame_t *frame, void *cookie,
                                xlator_t *this, int32_t op_ret,
-                               int32_t op_errno, dict_t *xattr)
+                               int32_t op_errno, dict_t *xattr, dict_t *xdata)
 {
         int             call_count = 0;
         afr_local_t     *local = NULL;
         afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+        int32_t         child_index = (long) cookie;
+
+        priv  = this->private;
+        local = frame->local;
+        sh    = &local->self_heal;
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Erasing of pending change "
+                        "log failed on %s for subvol %s, reason: %s",
+                        local->loc.path, priv->children[child_index]->name,
+                        strerror (op_errno));
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+        }
 
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
-                local = frame->local;
-                sh = &local->self_heal;
+                if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
+                        if (sh->old_loop_frame)
+                                sh_loop_finish (sh->old_loop_frame, this);
+                        sh->old_loop_frame = NULL;
+                        afr_sh_data_fail (frame, this);
+                        goto out;
+                }
                 if (!IA_ISREG (sh->type)) {
                         afr_sh_data_finish (frame, this);
                         goto out;
                 }
                 GF_ASSERT (sh->old_loop_frame);
-                afr_sh_data_lock (frame, this, 0, 0,
+                afr_sh_data_lock (frame, this, 0, 0, _gf_true, this->name,
                                   afr_post_sh_big_lock_success,
                                   afr_post_sh_big_lock_failure);
         }
@@ -382,74 +421,95 @@ out:
 int
 afr_sh_data_erase_pending (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              i = 0;
-        dict_t          **erase_xattr = NULL;
+        afr_sh_erase_pending (frame, this, AFR_DATA_TRANSACTION,
+                              afr_sh_data_erase_pending_cbk,
+                              afr_sh_data_finish);
+        return 0;
+}
+
+int
+afr_sh_data_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, struct iatt *pre,
+                       struct iatt *post, dict_t *xdata)
+{
+        afr_local_t     *local      = NULL;
+        afr_private_t   *priv       = NULL;
+        afr_self_heal_t *sh         = NULL;
+        int             call_count  = 0;
+        int             child_index = (long) cookie;
 
         local = frame->local;
-        sh = &local->self_heal;
         priv = this->private;
+        sh   = &local->self_heal;
+
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "%s: Failed to fsync on "
+                        "%s - %s", local->loc.path,
+                        priv->children[child_index]->name, strerror (op_errno));
+                LOCK (&frame->lock);
+                {
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                }
+                UNLOCK (&frame->lock);
+                if (sh->old_loop_frame)
+                        sh_loop_finish (sh->old_loop_frame, this);
+                sh->old_loop_frame = NULL;
+        }
 
-        afr_sh_pending_to_delta (priv, sh->xattr, sh->delta_matrix, sh->success,
-                                 priv->child_count, AFR_DATA_TRANSACTION);
-        gf_log (this->name, GF_LOG_DEBUG, "Delta matrix for: %s",
-                lkowner_utoa (&frame->root->lk_owner));
-        afr_sh_print_pending_matrix (sh->delta_matrix, this);
+        call_count = afr_frame_return (frame);
+        if (call_count == 0) {
+                if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC))
+                        afr_sh_data_fail (frame, this);
+                else
+                        afr_sh_data_erase_pending (frame, this);
+        }
+        return 0;
+}
 
-        erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count,
-                                 gf_afr_mt_dict_t);
+/*
+ * Before erasing xattrs, make sure the data is written to disk
+ */
+int
+afr_sh_data_fsync (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local     = NULL;
+        afr_private_t   *priv      = NULL;
+        afr_self_heal_t *sh        = NULL;
+        int             i          = 0;
+        int             call_count = 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->xattr[i]) {
-                        call_count++;
+        local = frame->local;
+        priv = this->private;
+        sh   = &local->self_heal;
 
-                        erase_xattr[i] = get_new_dict();
-                        dict_ref (erase_xattr[i]);
-                }
+        call_count        = sh->active_sinks;
+        if (call_count == 0) {
+                afr_sh_data_erase_pending (frame, this);
+                return 0;
         }
 
-        afr_sh_delta_to_xattr (priv, sh->delta_matrix, erase_xattr,
-                               priv->child_count, AFR_DATA_TRANSACTION);
-
-        GF_ASSERT (call_count);
         local->call_count = call_count;
         for (i = 0; i < priv->child_count; i++) {
-                if (!erase_xattr[i])
+                if (!sh->success[i] || sh->sources[i])
                         continue;
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "erasing pending flags from %s on %s",
-                        local->loc.path, priv->children[i]->name);
-
-                STACK_WIND_COOKIE (frame, afr_sh_data_erase_pending_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->fxattrop,
-                                   sh->healing_fd,
-                                   GF_XATTROP_ADD_ARRAY, erase_xattr[i]);
-                if (!--call_count)
-                        break;
+                STACK_WIND_COOKIE (frame, afr_sh_data_fsync_cbk,
+                                   (void *) (long) i, priv->children[i],
+                                   priv->children[i]->fops->fsync,
+                                   sh->healing_fd, 1, NULL);
         }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (erase_xattr[i]) {
-                        dict_unref (erase_xattr[i]);
-                }
-        }
-        GF_FREE (erase_xattr);
-
         return 0;
 }
 
-
 static struct afr_sh_algorithm *
 sh_algo_from_name (xlator_t *this, char *name)
 {
         int i = 0;
 
+        if (name == NULL)
+                goto out;
+
         while (afr_self_heal_algorithms[i].name) {
                 if (!strcmp (name, afr_self_heal_algorithms[i].name)) {
                         return &afr_self_heal_algorithms[i];
@@ -458,17 +518,22 @@ sh_algo_from_name (xlator_t *this, char *name)
                 i++;
         }
 
+out:
         return NULL;
 }
 
 
 static int
-sh_zero_byte_files_exist (afr_self_heal_t *sh, int child_count)
+sh_zero_byte_files_exist (afr_local_t *local, int child_count)
 {
-        int i;
-        int ret = 0;
+        int             i = 0;
+        int             ret = 0;
+        afr_self_heal_t *sh = NULL;
 
+        sh = &local->self_heal;
         for (i = 0; i < child_count; i++) {
+                if (!local->child_up[i] || sh->child_errno[i])
+                        continue;
                 if (sh->buf[i].ia_size == 0) {
                         ret = 1;
                         break;
@@ -495,8 +560,7 @@ afr_sh_data_pick_algo (call_frame_t *frame, xlator_t *this)
         if (algo == NULL) {
                 /* option not set, so fall back on heuristics */
 
-                if ((local->enoent_count != 0)
-                    || sh_zero_byte_files_exist (sh, priv->child_count)
+                if (sh_zero_byte_files_exist (local, priv->child_count)
                     || (sh->file_size <= (priv->data_self_heal_window_size *
                                           this->ctx->page_size))) {
 
@@ -534,11 +598,12 @@ afr_sh_data_sync_prepare (call_frame_t *frame, xlator_t *this)
         local = frame->local;
         sh = &local->self_heal;
 
-        sh->algo_completion_cbk = afr_sh_data_erase_pending;
+        sh->algo_completion_cbk = afr_sh_data_fsync;
         sh->algo_abort_cbk      = afr_sh_data_fail;
 
         sh_algo = afr_sh_data_pick_algo (frame, this);
 
+        sh->algo = sh_algo;
         sh_algo->fn (frame, this);
 
         return 0;
@@ -547,38 +612,46 @@ afr_sh_data_sync_prepare (call_frame_t *frame, xlator_t *this)
 int
 afr_sh_data_trim_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf)
+                      struct iatt *postbuf, dict_t *xdata)
 {
-        afr_private_t * priv = NULL;
-        afr_local_t * local  = NULL;
         int              call_count = 0;
         int              child_index = 0;
+        afr_private_t    *priv = NULL;
+        afr_local_t      *local  = NULL;
+        afr_self_heal_t  *sh = NULL;
 
-        priv = this->private;
+        priv  = this->private;
         local = frame->local;
+        sh    = &local->self_heal;
 
         child_index = (long) cookie;
 
         LOCK (&frame->lock);
         {
-                if (op_ret == -1)
-                        gf_log (this->name, GF_LOG_INFO,
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "ftruncate of %s on subvolume %s failed (%s)",
                                 local->loc.path,
                                 priv->children[child_index]->name,
                                 strerror (op_errno));
-                else
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                } else {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "ftruncate of %s on subvolume %s completed",
                                 local->loc.path,
                                 priv->children[child_index]->name);
+                }
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
-        if (call_count == 0)
-                afr_sh_data_sync_prepare (frame, this);
+        if (call_count == 0) {
+                if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC))
+                        afr_sh_data_fail (frame, this);
+                else
+                        afr_sh_data_sync_prepare (frame, this);
+        }
 
         return 0;
 }
@@ -612,7 +685,8 @@ afr_sh_data_trim_sinks (call_frame_t *frame, xlator_t *this)
                                    (void *) (long) i,
                                    priv->children[i],
                                    priv->children[i]->fops->ftruncate,
-                                   sh->healing_fd, sh->file_size);
+                                   sh->healing_fd, sh->file_size,
+                                   NULL);
 
                 if (!--call_count)
                         break;
@@ -626,6 +700,7 @@ afr_sh_inode_set_read_ctx (afr_self_heal_t *sh, xlator_t *this)
 {
         afr_private_t   *priv = NULL;
         int             ret = 0;
+        int             i = 0;
 
         priv = this->private;
         sh->source = afr_sh_select_source (sh->sources, priv->child_count);
@@ -634,6 +709,15 @@ afr_sh_inode_set_read_ctx (afr_self_heal_t *sh, xlator_t *this)
                 goto out;
         }
 
+        /* detect changes not visible through pending flags -- JIC */
+        for (i = 0; i < priv->child_count; i++) {
+                if (i == sh->source || sh->child_errno[i])
+                        continue;
+
+                if (SIZE_DIFFERS (&sh->buf[i], &sh->buf[sh->source]))
+                        sh->sources[i] = 0;
+        }
+
         afr_reset_children (sh->fresh_children, priv->child_count);
         afr_get_fresh_children (sh->success_children, sh->sources,
                                 sh->fresh_children, priv->child_count);
@@ -643,16 +727,254 @@ out:
         return ret;
 }
 
-int
+char*
+afr_get_sizes_str (afr_local_t *local, struct iatt *bufs, xlator_t *this)
+{
+        afr_private_t *priv = NULL;
+        int           i     = 0;
+        char          num[1024] = {0};
+        size_t        len = 0;
+        char          *sizes_str = NULL;
+        size_t        off = 0;
+        char          *fmt_str = "%llu bytes on %s, ";
+        char          *child_down =  " %s,";
+        char          *child_unknown = " %s,";
+        int           down_child_present = 0;
+        int           down_count = 0;
+        int           unknown_count = 0;
+        int           unknown_child_present = 0;
+        char          *down_subvol_1 = " down subvolume is ";
+        char          *unknown_subvol_1 = " unknown subvolume is ";
+        char          *down_subvol_2 = " down subvolumes are ";
+        char          *unknown_subvol_2 = " unknown subvolumes are ";
+
+        priv = this->private;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i] == 1) {
+                        len += snprintf (num, sizeof (num), fmt_str,
+                                         (unsigned long long) bufs[i].ia_size,
+                                         priv->children[i]->name);
+                } else if (local->child_up[i] == 0) {
+                        len += snprintf (num, sizeof (num), child_down,
+                                         priv->children[i]->name);
+                        if (!down_child_present)
+                                down_child_present = 1;
+                        down_count ++;
+                } else if (local->child_up[i] == -1) {
+                        len += snprintf (num, sizeof (num), child_unknown,
+                                         priv->children[i]->name);
+                        if (!unknown_child_present)
+                                unknown_child_present = 1;
+                        unknown_count++;
+                }
+
+        }
+
+        if (down_child_present) {
+                if (down_count > 1)
+                        len += snprintf (num, sizeof (num), "%s",
+                                         down_subvol_2);
+                else
+                        len += snprintf (num, sizeof (num), "%s",
+                                        down_subvol_1);
+        }
+        if (unknown_child_present) {
+                if (unknown_count > 1)
+                        len += snprintf (num, sizeof (num), "%s",
+                                         unknown_subvol_2);
+                else
+                        len += snprintf (num, sizeof (num), "%s",
+                                         unknown_subvol_1);
+        }
+
+        len++;//for '\0'
+
+        sizes_str = GF_CALLOC (len, sizeof (char), gf_common_mt_char);
+
+        if (!sizes_str)
+                return NULL;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i] == 1) {
+                        off += snprintf (sizes_str + off, len - off, fmt_str,
+                                         (unsigned long long) bufs[i].ia_size,
+                                         priv->children[i]->name);
+                }
+        }
+
+        if (down_child_present) {
+                if (down_count > 1) {
+                        off += snprintf (sizes_str + off, len - off, "%s",
+                                         down_subvol_2);
+                } else {
+                        off += snprintf (sizes_str + off, len - off, "%s",
+                                         down_subvol_1);
+                }
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i] == 0) {
+                        off += snprintf (sizes_str + off, len - off, child_down,
+                                         priv->children[i]->name);
+                }
+        }
+
+        if (unknown_child_present) {
+                if (unknown_count > 1) {
+                        off += snprintf (sizes_str + off, len - off, "%s",
+                                        unknown_subvol_2);
+                } else {
+                        off += snprintf (sizes_str + off, len - off, "%s",
+                                         unknown_subvol_1);
+                }
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i] == -1) {
+                        off += snprintf (sizes_str + off, len - off,
+                                         child_unknown,
+                                         priv->children[i]->name);
+
+                }
+        }
+
+        return sizes_str;
+}
+
+char*
+afr_get_sinks_str (xlator_t *this, afr_local_t *local, afr_self_heal_t *sh)
+{
+        afr_private_t   *priv = NULL;
+        int             i = 0;
+        char            num[1024] = {0};
+        size_t          len = 0;
+        char            *sinks_str = NULL;
+        char            *temp_str = " to sinks ";
+        char            *str_format = " %s,";
+        char            off = 0;
+
+        priv = this->private;
+
+        len += snprintf (num, sizeof (num), "%s", temp_str);
+        for (i = 0; i < priv->child_count; i++) {
+                if ((sh->sources[i] == 0) && (local->child_up[i] == 1)) {
+                        len += snprintf (num, sizeof (num), str_format,
+                                         priv->children[i]->name);
+                }
+        }
+
+        len ++;
+
+        sinks_str = GF_CALLOC (len, sizeof (char), gf_common_mt_char);
+
+        if (!sinks_str)
+                return NULL;
+
+        off += snprintf (sinks_str + off, len - off, "%s", temp_str);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if ((sh->sources[i] == 0) && (local->child_up[i] == 1)) {
+                        off += snprintf (sinks_str + off, len - off,
+                                         str_format,
+                                         priv->children[i]->name);
+                }
+        }
+
+        return sinks_str;
+
+}
+
+
+void
+afr_set_data_sh_info_str (afr_local_t *local, afr_self_heal_t *sh, xlator_t *this)
+{
+        char            *pending_matrix_str = NULL;
+        char            *sizes_str = NULL;
+        char            *sinks_str = NULL;
+        afr_private_t   *priv = NULL;
+
+        priv = this->private;
+
+        pending_matrix_str = afr_get_pending_matrix_str (sh->pending_matrix,
+                                                         this);
+        if (!pending_matrix_str)
+                pending_matrix_str = "";
+
+        sizes_str = afr_get_sizes_str (local, sh->buf, this);
+        if (!sizes_str)
+                sizes_str = "";
+
+        sinks_str = afr_get_sinks_str (this, local, sh);
+        if (!sinks_str)
+                sinks_str = "";
+
+        gf_asprintf (&sh->data_sh_info, " data self heal from %s %s with "
+                     "%s data %s", priv->children[sh->source]->name, sinks_str,
+                     sizes_str, pending_matrix_str);
+
+        if (pending_matrix_str && strcmp (pending_matrix_str, ""))
+                GF_FREE (pending_matrix_str);
+
+        if (sizes_str && strcmp (sizes_str, ""))
+                GF_FREE (sizes_str);
+}
+
+void
 afr_sh_data_fix (call_frame_t *frame, xlator_t *this)
 {
+        int              source = 0;
+        afr_local_t     *local      = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        source     = sh->source;
+        sh->block_size = this->ctx->page_size;
+        sh->file_size  = sh->buf[source].ia_size;
+
+        if (FILE_HAS_HOLES (&sh->buf[source]))
+                sh->file_has_holes = 1;
+
+        if (sh->background && sh->unwind && !sh->unwound) {
+                sh->unwind (sh->orig_frame, this, sh->op_ret, sh->op_errno,
+                            is_self_heal_failed (sh, AFR_CHECK_SPECIFIC));
+                sh->unwound = _gf_true;
+        }
+
+        afr_sh_mark_source_sinks (frame, this);
+        if (sh->active_sinks == 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "no active sinks for performing self-heal on file %s",
+                        local->loc.path);
+                afr_sh_data_finish (frame, this);
+                return;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "self-healing file %s from subvolume %s to %d other",
+                local->loc.path, priv->children[sh->source]->name,
+                sh->active_sinks);
+
+        sh->actual_sh_started = _gf_true;
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_SYNC_BEGIN);
+        afr_sh_data_trim_sinks (frame, this);
+}
+
+int
+afr_sh_data_fxattrop_fstat_done (call_frame_t *frame, xlator_t *this)
+{
         afr_local_t     *local      = NULL;
         afr_self_heal_t *sh = NULL;
         afr_private_t   *priv = NULL;
         int              nsources = 0;
-        int              source = 0;
-        int              i = 0;
         int              ret = 0;
+        int             *old_sources = NULL;
+        int             tstamp_source = 0;
+        int             i = 0;
 
         local = frame->local;
         sh = &local->self_heal;
@@ -660,19 +982,17 @@ afr_sh_data_fix (call_frame_t *frame, xlator_t *this)
 
         gf_log (this->name, GF_LOG_DEBUG, "Pending matrix for: %s",
                 lkowner_utoa (&frame->root->lk_owner));
+        if (sh->sync_done) {
+                //store sources before sync so that mtime can be set using the
+                //iatt buf from one of them.
+                old_sources = alloca (priv->child_count*sizeof (*old_sources));
+                memcpy (old_sources, sh->sources,
+                        priv->child_count * sizeof (*old_sources));
+        }
 
         nsources = afr_build_sources (this, sh->xattr, sh->buf, sh->pending_matrix,
                                       sh->sources, sh->success_children,
-                                      AFR_DATA_TRANSACTION, NULL, _gf_false);
-        if (nsources == 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "No self-heal needed for %s",
-                        local->loc.path);
-
-                afr_sh_data_finish (frame, this);
-                return 0;
-        }
-
+                                      AFR_DATA_TRANSACTION, NULL, _gf_true);
         if ((nsources == -1)
             && (priv->favorite_child != -1)
             && (sh->child_errno[priv->favorite_child] == 0)) {
@@ -690,17 +1010,16 @@ afr_sh_data_fix (call_frame_t *frame, xlator_t *this)
         }
 
         if (nsources == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to self-heal contents of '%s' (possible "
-                        "split-brain). Please delete the file from all but "
-                        "the preferred subvolume.", local->loc.path);
-
-                local->govinda_gOvinda = 1;
+                afr_sh_print_split_brain_log (sh->pending_matrix, this,
+                                              local->loc.path);
+                afr_set_split_brain (this, sh->inode, DONT_KNOW, SPB);
 
                 afr_sh_data_fail (frame, this);
                 return 0;
         }
 
+        afr_set_split_brain (this, sh->inode, DONT_KNOW, NO_SPB);
+
         ret = afr_sh_inode_set_read_ctx (sh, this);
         if (ret) {
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -710,49 +1029,42 @@ afr_sh_data_fix (call_frame_t *frame, xlator_t *this)
                 return 0;
         }
 
-        source     = sh->source;
-        sh->block_size = this->ctx->page_size;
-        sh->file_size  = sh->buf[source].ia_size;
-
-        if (FILE_HAS_HOLES (&sh->buf[source]))
-                sh->file_has_holes = 1;
-
-        /* detect changes not visible through pending flags -- JIC */
-        for (i = 0; i < priv->child_count; i++) {
-                if (i == source || sh->child_errno[i])
-                        continue;
-
-                if (SIZE_DIFFERS (&sh->buf[i], &sh->buf[source]))
-                        sh->sources[i] = 0;
-        }
+        if (sh->sync_done) {
+                /* Perform setattr from one of the old_sources if possible
+                 * Because only they have the correct mtime, the new sources
+                 * (i.e. old sinks) have mtime from last writev in sync.
+                 */
+                tstamp_source = sh->source;
+                for (i = 0; i < priv->child_count; i++) {
+                        if (old_sources[i] && sh->sources[i])
+                                tstamp_source = i;
+                }
+                afr_sh_data_setattr (frame, this, &sh->buf[tstamp_source]);
+        } else {
+                afr_set_data_sh_info_str (local, sh, this);
+                if (nsources == 0) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "No self-heal needed for %s",
+                                local->loc.path);
 
-        if (sh->background && sh->unwind) {
-                sh->unwind (sh->orig_frame, this, sh->op_ret, sh->op_errno);
-                sh->unwound = _gf_true;
-        }
+                        afr_sh_data_finish (frame, this);
+                        return 0;
+                }
 
-        afr_sh_mark_source_sinks (frame, this);
-        if (sh->active_sinks == 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "no active sinks for performing self-heal on file %s",
-                        local->loc.path);
-                afr_sh_data_finish (frame, this);
-                return 0;
+                if (sh->do_data_self_heal &&
+                    afr_data_self_heal_enabled (priv->data_self_heal))
+                        afr_sh_data_fix (frame, this);
+                else
+                        afr_sh_data_finish (frame, this);
         }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "self-healing file %s from subvolume %s to %d other",
-                local->loc.path, priv->children[sh->source]->name,
-                sh->active_sinks);
-        afr_sh_data_trim_sinks (frame, this);
-
         return 0;
 }
 
 int
 afr_lookup_select_read_child_by_txn_type (xlator_t *this, afr_local_t *local,
                                           dict_t **xattr,
-                                          afr_transaction_type txn_type)
+                                          afr_transaction_type txn_type,
+                                          uuid_t gfid)
 {
         afr_private_t            *priv      = NULL;
         int                      read_child = -1;
@@ -769,20 +1081,33 @@ afr_lookup_select_read_child_by_txn_type (xlator_t *this, afr_local_t *local,
         bufs = local->cont.lookup.bufs;
         success_children = local->cont.lookup.success_children;
 
-        pending_matrix = afr_matrix_create (priv->child_count,
-                                            priv->child_count);
-        if (NULL == pending_matrix)
-                goto out;
-
+        pending_matrix = local->cont.lookup.pending_matrix;
         sources = local->cont.lookup.sources;
         memset (sources, 0, sizeof (*sources) * priv->child_count);
 
         nsources = afr_build_sources (this, xattr, bufs, pending_matrix,
                                       sources, success_children, txn_type,
                                       &subvol_status, _gf_false);
-        if (subvol_status & SPLIT_BRAIN)
-                gf_log (this->name, GF_LOG_WARNING, "%s: Possible split-brain",
+        if (subvol_status & SPLIT_BRAIN) {
+                gf_log (this->name, GF_LOG_DEBUG, "%s: Possible split-brain",
                         local->loc.path);
+                switch (txn_type) {
+                case AFR_DATA_TRANSACTION:
+                        local->cont.lookup.possible_spb = _gf_true;
+                        nsources = 1;
+                        sources[success_children[0]] = 1;
+                        break;
+                case AFR_ENTRY_TRANSACTION:
+                        read_child = afr_get_no_xattr_dir_read_child (this,
+                                                             success_children,
+                                                             bufs);
+                        sources[read_child] = 1;
+                        nsources = 1;
+                        break;
+                default:
+                        break;
+                }
+        }
         if (nsources < 0)
                 goto out;
 
@@ -792,39 +1117,18 @@ afr_lookup_select_read_child_by_txn_type (xlator_t *this, afr_local_t *local,
                                                         priv->child_count,
                                                         prev_read_child,
                                                         config_read_child,
-                                                        sources);
+                                                        sources,
+                                                        priv->hash_mode, gfid);
 out:
-        afr_matrix_cleanup (pending_matrix, priv->child_count);
         gf_log (this->name, GF_LOG_DEBUG, "returning read_child: %d",
                 read_child);
         return read_child;
 }
 
 int
-afr_sh_data_special_file_fix (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t   *priv = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_local_t     *local = NULL;
-        int             i = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count ; i++)
-                if (1 == local->child_up[i])
-                        sh->success[i] = 1;
-
-        afr_sh_data_erase_pending (frame, this);
-
-        return 0;
-}
-
-int
 afr_sh_data_fstat_cbk (call_frame_t *frame, void *cookie,
                        xlator_t *this, int32_t op_ret, int32_t op_errno,
-                       struct iatt *buf)
+                       struct iatt *buf, dict_t *xdata)
 {
         afr_private_t   *priv  = NULL;
         afr_local_t     *local = NULL;
@@ -847,6 +1151,12 @@ afr_sh_data_fstat_cbk (call_frame_t *frame, void *cookie,
                         sh->buf[child_index] = *buf;
                         sh->success_children[sh->success_count] = child_index;
                         sh->success_count++;
+                } else {
+                        gf_log (this->name, GF_LOG_ERROR, "%s: fstat failed "
+                                "on %s, reason %s", local->loc.path,
+                                priv->children[child_index]->name,
+                                strerror (op_errno));
+                        sh->child_errno[child_index] = op_errno;
                 }
         }
         UNLOCK (&frame->lock);
@@ -857,13 +1167,17 @@ afr_sh_data_fstat_cbk (call_frame_t *frame, void *cookie,
                 /* Previous versions of glusterfs might have set
                  * the pending data xattrs which need to be erased
                  */
-                if (IA_ISREG (buf->ia_type))
-                        afr_sh_data_fix (frame, this);
-                else
-                        afr_sh_data_special_file_fix (frame, this);
-
+                if (!afr_sh_data_proceed (sh->success_count)) {
+                        gf_log (this->name, GF_LOG_ERROR, "inspecting metadata "
+                                "succeeded on < %d children, aborting "
+                                "self-heal for %s", AFR_SH_MIN_PARTICIPANTS,
+                                local->loc.path);
+                        afr_sh_data_fail (frame, this);
+                        goto out;
+                }
+                afr_sh_data_fxattrop_fstat_done (frame, this);
         }
-
+out:
         return 0;
 }
 
@@ -874,33 +1188,41 @@ afr_sh_data_fstat (call_frame_t *frame, xlator_t *this)
         afr_self_heal_t *sh    = NULL;
         afr_local_t     *local = NULL;
         afr_private_t   *priv  = NULL;
-        int call_count = 0;
-        int i = 0;
+        int             call_count = 0;
+        int             i = 0;
+        int             child = 0;
+        int32_t         *fstat_children = NULL;
 
         priv  = this->private;
         local = frame->local;
         sh    = &local->self_heal;
 
-        call_count = afr_up_children_count (local->child_up,
-                                            priv->child_count);
-
+        fstat_children = memdup (sh->success_children,
+                                 sizeof (*fstat_children) * priv->child_count);
+        if (!fstat_children) {
+                afr_sh_data_fail (frame, this);
+                goto out;
+        }
+        call_count = sh->success_count;
         local->call_count = call_count;
 
+        memset (sh->buf, 0, sizeof (*sh->buf) * priv->child_count);
         afr_reset_children (sh->success_children, priv->child_count);
         sh->success_count = 0;
         for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_sh_data_fstat_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fstat,
-                                           sh->healing_fd);
-
-                        if (!--call_count)
-                                break;
-                }
+                child = fstat_children[i];
+                if (child == -1)
+                        break;
+                STACK_WIND_COOKIE (frame, afr_sh_data_fstat_cbk,
+                                   (void *) (long) child,
+                                   priv->children[child],
+                                   priv->children[child]->fops->fstat,
+                                   sh->healing_fd, NULL);
+                --call_count;
         }
-
+        GF_ASSERT (!call_count);
+out:
+        GF_FREE (fstat_children);
         return 0;
 }
 
@@ -929,73 +1251,60 @@ afr_sh_common_fxattrop_resp_handler (call_frame_t *frame, void *cookie,
                         sh->xattr[child_index] = dict_ref (xattr);
                         sh->success_children[sh->success_count] = child_index;
                         sh->success_count++;
+                } else {
+                        gf_log (this->name, GF_LOG_ERROR, "fxattrop of %s "
+                                "failed on %s, reason %s", local->loc.path,
+                                priv->children[child_index]->name,
+                                strerror (op_errno));
+                        sh->child_errno[child_index] = op_errno;
                 }
         }
         UNLOCK (&frame->lock);
 }
 
 int
-afr_post_sh_data_fxattrop_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret, int32_t op_errno,
-                               dict_t *xattr)
+afr_sh_data_fxattrop_cbk (call_frame_t *frame, void *cookie,
+                          xlator_t *this, int32_t op_ret, int32_t op_errno,
+                          dict_t *xattr, dict_t *xdata)
 {
         int             call_count  = -1;
-        int             ret = 0;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        afr_sh_common_fxattrop_resp_handler (frame, cookie, this, op_ret,
-                                             op_errno, xattr);
+        afr_local_t     *local  = NULL;
+        afr_self_heal_t *sh     = NULL;
 
         local = frame->local;
-        sh = &local->self_heal;
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-                (void) afr_build_sources (this, sh->xattr, NULL,
-                                          sh->pending_matrix,
-                                          sh->sources, sh->success_children,
-                                          AFR_DATA_TRANSACTION, NULL, _gf_false);
-                ret = afr_sh_inode_set_read_ctx (sh, this);
-                if (ret)
-                        afr_sh_data_fail (frame, this);
-                else
-                        afr_sh_set_timestamps (frame, this);
-        }
-
-        return 0;
-}
-
-int
-afr_sh_data_fxattrop_cbk (call_frame_t *frame, void *cookie,
-                          xlator_t *this, int32_t op_ret, int32_t op_errno,
-                          dict_t *xattr)
-{
-        int call_count  = -1;
+        sh    = &local->self_heal;
 
         afr_sh_common_fxattrop_resp_handler (frame, cookie, this, op_ret,
                                              op_errno, xattr);
 
         call_count = afr_frame_return (frame);
         if (call_count == 0) {
+                if (!afr_sh_data_proceed (sh->success_count)) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s, inspecting "
+                                "change log succeeded on < %d children",
+                                local->loc.path, AFR_SH_MIN_PARTICIPANTS);
+                        afr_sh_data_fail (frame, this);
+                        goto out;
+                }
                 afr_sh_data_fstat (frame, this);
         }
-
+out:
         return 0;
 }
 
 
 int
-afr_sh_data_fxattrop (call_frame_t *frame, xlator_t *this,
-                      afr_fxattrop_cbk_t fxattrop_cbk)
+afr_sh_data_fxattrop (call_frame_t *frame, xlator_t *this)
 {
         afr_self_heal_t *sh    = NULL;
         afr_local_t     *local = NULL;
         afr_private_t   *priv  = NULL;
-        dict_t          *xattr_req = NULL;
+        dict_t          **xattr_req;
         int32_t         *zero_pending = NULL;
         int call_count = 0;
         int i = 0;
         int ret = 0;
+	int j;
 
         priv  = this->private;
         local = frame->local;
@@ -1006,42 +1315,53 @@ afr_sh_data_fxattrop (call_frame_t *frame, xlator_t *this,
 
         local->call_count = call_count;
 
-        xattr_req = dict_new();
-        if (!xattr_req) {
-                ret = -1;
-                goto out;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                zero_pending = GF_CALLOC (3, sizeof (*zero_pending),
-                                          gf_afr_mt_int32_t);
-                if (!zero_pending) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_dynptr (xattr_req, priv->pending_key[i],
-                                       zero_pending,
-                                       3 * sizeof (*zero_pending));
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Unable to set dict value");
-                        goto out;
-                } else {
-                        zero_pending = NULL;
-                }
-        }
+	xattr_req = GF_CALLOC(priv->child_count, sizeof(struct dict_t *),
+			      gf_afr_mt_dict_t);
+	if (!xattr_req)
+		goto out;
+
+	for (i = 0; i < priv->child_count; i++) {
+		xattr_req[i] = dict_new();
+		if (!xattr_req[i]) {
+			ret = -1;
+			goto out;
+		}
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		for (j = 0; j < priv->child_count; j++) {
+			zero_pending = GF_CALLOC (3, sizeof (*zero_pending),
+						  gf_afr_mt_int32_t);
+			if (!zero_pending) {
+				ret = -1;
+				goto out;
+			}
+			ret = dict_set_dynptr (xattr_req[i], priv->pending_key[j],
+					       zero_pending,
+					       3 * sizeof (*zero_pending));
+			if (ret < 0) {
+				gf_log (this->name, GF_LOG_WARNING,
+					"Unable to set dict value");
+				goto out;
+			} else {
+				zero_pending = NULL;
+			}
+		}
+	}
 
         afr_reset_xattr (sh->xattr, priv->child_count);
         afr_reset_children (sh->success_children, priv->child_count);
+        memset (sh->child_errno, 0,
+                sizeof (*sh->child_errno) * priv->child_count);
         sh->success_count = 0;
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, fxattrop_cbk,
+                        STACK_WIND_COOKIE (frame, afr_sh_data_fxattrop_cbk,
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->fxattrop,
                                            sh->healing_fd, GF_XATTROP_ADD_ARRAY,
-                                           xattr_req);
+                                           xattr_req[i], NULL);
 
                         if (!--call_count)
                                 break;
@@ -1049,14 +1369,16 @@ afr_sh_data_fxattrop (call_frame_t *frame, xlator_t *this,
         }
 
 out:
-        if (xattr_req)
-                dict_unref (xattr_req);
+	if (xattr_req) {
+		for (i = 0; i < priv->child_count; i++)
+			if (xattr_req[i])
+				dict_unref(xattr_req[i]);
+		GF_FREE(xattr_req);
+	}
 
         if (ret) {
-                if (zero_pending)
-                        GF_FREE (zero_pending);
-                sh->op_failed = 1;
-                afr_sh_data_done (frame, this);
+                GF_FREE (zero_pending);
+                afr_sh_data_fail (frame, this);
         }
 
         return 0;
@@ -1072,7 +1394,23 @@ afr_sh_data_big_lock_success (call_frame_t *frame, xlator_t *this)
         sh = &local->self_heal;
 
         sh->data_lock_held = _gf_true;
-        afr_sh_data_fxattrop (frame, this, afr_sh_data_fxattrop_cbk);
+        afr_sh_data_fxattrop (frame, this);
+        return 0;
+}
+
+int
+afr_sh_dom_lock_success (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t   *local = NULL;
+        afr_self_heal_t *sh = NULL;
+
+        local = frame->local;
+        sh = &local->self_heal;
+
+        sh->sh_dom_lock_held = _gf_true;
+        afr_sh_data_lock (frame, this, 0, 0, _gf_true, this->name,
+                          afr_sh_data_big_lock_success,
+                          afr_sh_data_fail);
         return 0;
 }
 
@@ -1121,8 +1459,13 @@ afr_sh_data_post_nonblocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
                         "failed for %s. by %s",
                         local->loc.path, lkowner_utoa (&frame->root->lk_owner));
 
-                int_lock->lock_cbk = afr_sh_data_post_blocking_inodelk_cbk;
-                afr_blocking_lock (frame, this);
+		if (!sh->data_lock_block) {
+			sh->data_lock_failure_handler(frame, this);
+		} else {
+			int_lock->lock_cbk =
+				afr_sh_data_post_blocking_inodelk_cbk;
+			afr_blocking_lock (frame, this);
+		}
         } else {
 
                 gf_log (this->name, GF_LOG_DEBUG, "Non Blocking data inodelks "
@@ -1135,9 +1478,11 @@ afr_sh_data_post_nonblocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
 }
 
 int
-afr_sh_data_lock_rec (call_frame_t *frame, xlator_t *this, off_t start, off_t len)
+afr_sh_data_lock_rec (call_frame_t *frame, xlator_t *this, char *dom,
+                      off_t start, off_t len)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
 
         local    = frame->local;
@@ -1148,11 +1493,14 @@ afr_sh_data_lock_rec (call_frame_t *frame, xlator_t *this, off_t start, off_t le
 
         afr_set_lock_number (frame, this);
 
-        int_lock->lk_flock.l_start = start;
-        int_lock->lk_flock.l_len   = len;
-        int_lock->lk_flock.l_type  = F_WRLCK;
         int_lock->lock_cbk         = afr_sh_data_post_nonblocking_inodelk_cbk;
 
+        int_lock->domain = dom;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+        inodelk->flock.l_start = start;
+        inodelk->flock.l_len   = len;
+        inodelk->flock.l_type  = F_WRLCK;
+
         afr_nonblocking_inodelk (frame, this);
 
         return 0;
@@ -1171,7 +1519,8 @@ afr_post_sh_big_lock_success (call_frame_t *frame, xlator_t *this)
         sh_loop_finish (sh->old_loop_frame, this);
         sh->old_loop_frame = NULL;
         sh->data_lock_held = _gf_true;
-        afr_sh_data_fxattrop (frame, this, afr_post_sh_data_fxattrop_cbk);
+        sh->sync_done = _gf_true;
+        afr_sh_data_fxattrop (frame, this);
         return 0;
 }
 
@@ -1194,8 +1543,8 @@ afr_post_sh_big_lock_failure (call_frame_t *frame, xlator_t *this)
 
 int
 afr_sh_data_lock (call_frame_t *frame, xlator_t *this,
-                  off_t start, off_t len,
-                  afr_lock_cbk_t success_handler,
+                  off_t start, off_t len, gf_boolean_t block,
+                  char *dom, afr_lock_cbk_t success_handler,
                   afr_lock_cbk_t failure_handler)
 {
         afr_local_t *   local = NULL;
@@ -1206,12 +1555,13 @@ afr_sh_data_lock (call_frame_t *frame, xlator_t *this,
 
         sh->data_lock_success_handler = success_handler;
         sh->data_lock_failure_handler = failure_handler;
-        return afr_sh_data_lock_rec (frame, this, start, len);
+	sh->data_lock_block = block;
+        return afr_sh_data_lock_rec (frame, this, dom, start, len);
 }
 
 int
 afr_sh_data_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, fd_t *fd)
+                      int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         afr_local_t     *local = NULL;
         afr_self_heal_t *sh = NULL;
@@ -1237,20 +1587,20 @@ afr_sh_data_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                 local->loc.path,
                                 priv->children[child_index]->name,
                                 strerror (op_errno));
-                        sh->op_failed = 1;
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                } else {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "open of %s succeeded on child %s",
+                                local->loc.path,
+                                priv->children[child_index]->name);
                 }
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "open of %s succeeded on child %s",
-                        local->loc.path,
-                        priv->children[child_index]->name);
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
-                if (sh->op_failed) {
+                if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
                         afr_sh_data_fail (frame, this);
                         return 0;
                 }
@@ -1259,9 +1609,8 @@ afr_sh_data_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         "fd for %s opened, commencing sync",
                         local->loc.path);
 
-                afr_sh_data_lock (frame, this, 0, 0,
-                                  afr_sh_data_big_lock_success,
-                                  afr_sh_data_fail);
+                afr_sh_data_lock (frame, this, 0, 0, _gf_true, priv->sh_domain,
+                                  afr_sh_dom_lock_success, afr_sh_data_fail);
         }
 
         return 0;
@@ -1298,7 +1647,7 @@ afr_sh_data_open (call_frame_t *frame, xlator_t *this)
                                    priv->children[i],
                                    priv->children[i]->fops->open,
                                    &local->loc,
-                                   O_RDWR|O_LARGEFILE, fd, 0);
+                                   O_RDWR|O_LARGEFILE, fd, NULL);
 
                 if (!--call_count)
                         break;
@@ -1307,20 +1656,93 @@ afr_sh_data_open (call_frame_t *frame, xlator_t *this)
         return 0;
 }
 
+void
+afr_sh_non_reg_fix (call_frame_t *frame, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno)
+{
+        afr_private_t   *priv = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_local_t     *local = NULL;
+        int             i = 0;
+
+        if (op_ret < 0) {
+                afr_sh_data_fail (frame, this);
+                return;
+        }
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        for (i = 0; i < priv->child_count ; i++) {
+                if (1 == local->child_up[i])
+                        sh->success[i] = 1;
+        }
+
+        afr_sh_erase_pending (frame, this, AFR_DATA_TRANSACTION,
+                              afr_sh_data_erase_pending_cbk,
+                              afr_sh_data_finish);
+}
 
 int
-afr_self_heal_data (call_frame_t *frame, xlator_t *this)
+afr_sh_non_reg_lock_success (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t   *local = NULL;
         afr_self_heal_t *sh = NULL;
-        afr_private_t *priv = this->private;
 
         local = frame->local;
         sh = &local->self_heal;
+        sh->data_lock_held = _gf_true;
+        afr_sh_common_lookup (frame, this, &local->loc,
+                              afr_sh_non_reg_fix, NULL,
+                              AFR_LOOKUP_FAIL_CONFLICTS |
+                              AFR_LOOKUP_FAIL_MISSING_GFIDS,
+                              NULL);
+        return 0;
+}
 
+gf_boolean_t
+afr_can_start_data_self_heal (afr_self_heal_t *sh, afr_private_t *priv)
+{
+        if (sh->force_confirm_spb)
+                return _gf_true;
         if (sh->do_data_self_heal &&
-            afr_data_self_heal_enabled (priv->data_self_heal)) {
-                afr_sh_data_open (frame, this);
+            afr_data_self_heal_enabled (priv->data_self_heal))
+                return _gf_true;
+        return _gf_false;
+}
+
+int
+afr_self_heal_data (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh    = NULL;
+        afr_private_t   *priv  = this->private;
+        int             ret    = -1;
+
+        local = frame->local;
+        sh = &local->self_heal;
+
+        sh->sh_type_in_action = AFR_SELF_HEAL_DATA;
+
+        if (afr_can_start_data_self_heal (sh, priv)) {
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_STARTED);
+                ret = afr_inodelk_init (&local->internal_lock.inodelk[1],
+                                        priv->sh_domain, priv->child_count);
+                if (ret < 0) {
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                        afr_sh_data_done (frame, this);
+                        return 0;
+                }
+
+                if (IA_ISREG (sh->type)) {
+                        afr_sh_data_open (frame, this);
+                } else {
+                        afr_sh_data_lock (frame, this, 0, 0, _gf_true,
+                                          this->name,
+                                          afr_sh_non_reg_lock_success,
+                                          afr_sh_data_fail);
+                }
         } else {
                 gf_log (this->name, GF_LOG_TRACE,
                         "not doing data self heal on %s",
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
index 3aefd3b37..53491a1d7 100644
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -111,7 +102,7 @@ afr_sh_entry_finish (call_frame_t *frame, xlator_t *this)
 int
 afr_sh_entry_erase_pending_cbk (call_frame_t *frame, void *cookie,
                                 xlator_t *this, int32_t op_ret,
-                                int32_t op_errno, dict_t *xattr)
+                                int32_t op_errno, dict_t *xattr, dict_t *xdata)
 {
         long                 i          = 0;
         int                  call_count = 0;
@@ -166,72 +157,20 @@ afr_sh_entry_erase_pending (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t     *local = NULL;
         afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              i = 0;
-        dict_t          **erase_xattr = NULL;
-        int              need_unwind = 0;
 
         local = frame->local;
         sh = &local->self_heal;
-        priv = this->private;
 
         if (sh->entries_skipped) {
-                need_unwind = 1;
-                sh->op_failed = _gf_true;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 goto out;
         }
-        afr_sh_pending_to_delta (priv, sh->xattr, sh->delta_matrix, sh->success,
-                                 priv->child_count, AFR_ENTRY_TRANSACTION);
-
-        erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count,
-                                 gf_afr_mt_dict_t);
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->xattr[i]) {
-                        call_count++;
-
-                        erase_xattr[i] = get_new_dict();
-                        dict_ref (erase_xattr[i]);
-                }
-        }
-
-        if (call_count == 0)
-                need_unwind = 1;
-
-        afr_sh_delta_to_xattr (priv, sh->delta_matrix, erase_xattr,
-                               priv->child_count, AFR_ENTRY_TRANSACTION);
-
-        local->call_count = call_count;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!erase_xattr[i])
-                        continue;
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "erasing pending flags from %s on %s",
-                        local->loc.path, priv->children[i]->name);
-
-                STACK_WIND_COOKIE (frame, afr_sh_entry_erase_pending_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->xattrop,
-                                   &local->loc,
-                                   GF_XATTROP_ADD_ARRAY, erase_xattr[i]);
-                if (!--call_count)
-                        break;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (erase_xattr[i]) {
-                        dict_unref (erase_xattr[i]);
-                }
-        }
-        GF_FREE (erase_xattr);
-
+        afr_sh_erase_pending (frame, this, AFR_ENTRY_TRANSACTION,
+                              afr_sh_entry_erase_pending_cbk,
+                              afr_sh_entry_finish);
+        return 0;
 out:
-        if (need_unwind)
-                afr_sh_entry_finish (frame, this);
-
+        afr_sh_entry_finish (frame, this);
         return 0;
 }
 
@@ -344,7 +283,8 @@ int
 afr_sh_entry_expunge_parent_setattr_cbk (call_frame_t *expunge_frame,
                                          void *cookie, xlator_t *this,
                                          int32_t op_ret, int32_t op_errno,
-                                         struct iatt *preop, struct iatt *postop)
+                                         struct iatt *preop, struct iatt *postop,
+                                         dict_t *xdata)
 {
         afr_private_t   *priv          = NULL;
         afr_local_t     *expunge_local = NULL;
@@ -380,7 +320,7 @@ afr_sh_entry_expunge_remove_cbk (call_frame_t *expunge_frame, void *cookie,
                                  xlator_t *this,
                                  int32_t op_ret, int32_t op_errno,
                                  struct iatt *preparent,
-                                 struct iatt *postparent)
+                                 struct iatt *postparent, dict_t *xdata)
 {
         afr_private_t   *priv = NULL;
         afr_local_t     *expunge_local = NULL;
@@ -415,7 +355,7 @@ afr_sh_entry_expunge_remove_cbk (call_frame_t *expunge_frame, void *cookie,
                            priv->children[active_src]->fops->setattr,
                            &expunge_sh->parent_loc,
                            &expunge_sh->parentbuf,
-                           valid);
+                           valid, NULL);
 
         return 0;
 }
@@ -439,7 +379,7 @@ afr_sh_entry_expunge_unlink (call_frame_t *expunge_frame, xlator_t *this,
                            (void *) (long) active_src,
                            priv->children[active_src],
                            priv->children[active_src]->fops->unlink,
-                           &expunge_local->loc);
+                           &expunge_local->loc, 0, NULL);
 
         return 0;
 }
@@ -464,7 +404,7 @@ afr_sh_entry_expunge_rmdir (call_frame_t *expunge_frame, xlator_t *this,
                            (void *) (long) active_src,
                            priv->children[active_src],
                            priv->children[active_src]->fops->rmdir,
-                           &expunge_local->loc, 1);
+                           &expunge_local->loc, 1, NULL);
 
         return 0;
 }
@@ -588,7 +528,7 @@ afr_sh_entry_expunge_purge (call_frame_t *expunge_frame, xlator_t *this,
                            (void *) (long) active_src,
                            priv->children[active_src],
                            priv->children[active_src]->fops->lookup,
-                           &expunge_local->loc, 0);
+                           &expunge_local->loc, NULL);
 
         return 0;
 }
@@ -640,7 +580,8 @@ afr_sh_entry_expunge_entry_cbk (call_frame_t *expunge_frame, void *cookie,
 
         if (need_expunge) {
                 gf_log (this->name, GF_LOG_INFO,
-                        "missing entry %s on %s",
+                        "Entry %s is missing on %s and deleting from "
+                        "replica's other bricks",
                         expunge_local->loc.path,
                         priv->children[source]->name);
 
@@ -672,6 +613,19 @@ out:
         return 0;
 }
 
+static gf_boolean_t
+can_skip_entry_self_heal (char *name, loc_t *parent_loc)
+{
+        if (strcmp (name, ".") == 0) {
+                return _gf_true;
+        } else if (strcmp (name, "..") == 0) {
+                return _gf_true;
+        } else if (loc_is_root (parent_loc) &&
+                   (strcmp (name, GF_REPLICATE_TRASH_DIR) == 0)) {
+                return _gf_true;
+        }
+        return _gf_false;
+}
 
 int
 afr_sh_entry_expunge_entry (call_frame_t *frame, xlator_t *this,
@@ -699,15 +653,7 @@ afr_sh_entry_expunge_entry (call_frame_t *frame, xlator_t *this,
         sh->expunge_done = afr_sh_entry_expunge_entry_done;
 
         name = entry->d_name;
-
-        if ((strcmp (name, ".") == 0)
-            || (strcmp (name, "..") == 0)
-            || ((strcmp (local->loc.path, "/") == 0)
-                && (strcmp (name, GF_REPLICATE_TRASH_DIR) == 0))) {
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "skipping inspection of %s under %s",
-                        name, local->loc.path);
+        if (can_skip_entry_self_heal (name, &local->loc)) {
                 op_ret = 0;
                 goto out;
         }
@@ -747,7 +693,7 @@ afr_sh_entry_expunge_entry (call_frame_t *frame, xlator_t *this,
                            (void *) (long) source,
                            priv->children[source],
                            priv->children[source]->fops->lookup,
-                           &expunge_local->loc, 0);
+                           &expunge_local->loc, NULL);
 
         ret = 0;
 out:
@@ -762,7 +708,7 @@ int
 afr_sh_entry_expunge_readdir_cbk (call_frame_t *frame, void *cookie,
                                   xlator_t *this,
                                   int32_t op_ret, int32_t op_errno,
-                                  gf_dirent_t *entries)
+                                  gf_dirent_t *entries, dict_t *xdata)
 {
         afr_private_t   *priv = NULL;
         afr_local_t     *local  = NULL;
@@ -860,7 +806,7 @@ afr_sh_entry_expunge_all (call_frame_t *frame, xlator_t *this)
         active_src = next_active_sink (frame, this, sh->active_source);
         sh->active_source = active_src;
 
-        if (sh->op_failed) {
+        if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
                 goto out;
         }
 
@@ -923,7 +869,8 @@ int
 afr_sh_entry_impunge_setattr_cbk (call_frame_t *impunge_frame, void *cookie,
                                   xlator_t *this,
                                   int32_t op_ret, int32_t op_errno,
-                                  struct iatt *preop, struct iatt *postop)
+                                  struct iatt *preop, struct iatt *postop,
+                                  dict_t *xdata)
 {
         int              call_count = 0;
         afr_private_t   *priv = NULL;
@@ -960,7 +907,8 @@ int
 afr_sh_entry_impunge_parent_setattr_cbk (call_frame_t *setattr_frame,
                                          void *cookie, xlator_t *this,
                                          int32_t op_ret, int32_t op_errno,
-                                         struct iatt *preop, struct iatt *postop)
+                                         struct iatt *preop, struct iatt *postop,
+                                         dict_t *xdata)
 {
         int             call_count = 0;
         afr_local_t     *setattr_local = NULL;
@@ -1024,7 +972,7 @@ afr_sh_entry_impunge_setattr (call_frame_t *impunge_frame, xlator_t *this)
                                    (void *) (long) i, priv->children[i],
                                    priv->children[i]->fops->setattr,
                                    &setattr_local->loc,
-                                   &impunge_sh->parentbuf, valid);
+                                   &impunge_sh->parentbuf, valid, NULL);
 
                 valid = GF_SET_ATTR_UID   | GF_SET_ATTR_GID |
                         GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
@@ -1033,7 +981,7 @@ afr_sh_entry_impunge_setattr (call_frame_t *impunge_frame, xlator_t *this)
                                    (void *) (long) i, priv->children[i],
                                    priv->children[i]->fops->setattr,
                                    &impunge_local->loc,
-                                   &impunge_sh->entrybuf, valid);
+                                   &impunge_sh->entrybuf, valid, NULL);
                 call_count--;
         }
         GF_ASSERT (!call_count);
@@ -1049,7 +997,7 @@ int
 afr_sh_entry_impunge_xattrop_cbk (call_frame_t *impunge_frame, void *cookie,
                                   xlator_t *this,
                                   int32_t op_ret, int32_t op_errno,
-                                  dict_t *xattr)
+                                  dict_t *xattr, dict_t *xdata)
 {
         afr_private_t   *priv = NULL;
         afr_local_t     *impunge_local = NULL;
@@ -1077,33 +1025,6 @@ out:
         return 0;
 }
 
-void
-afr_sh_prepare_new_entry_pending_matrix (int32_t **pending,
-                                         int *child_errno,
-                                         struct iatt *buf,
-                                         unsigned int child_count)
-{
-        int              midx             = 0;
-        int              idx              = 0;
-        int              i                = 0;
-
-        midx = afr_index_for_transaction_type (AFR_METADATA_TRANSACTION);
-        if (IA_ISDIR (buf->ia_type))
-                idx = afr_index_for_transaction_type (AFR_ENTRY_TRANSACTION);
-        else if (IA_ISREG (buf->ia_type))
-                idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
-        else
-                idx = -1;
-        for (i = 0; i < child_count; i++) {
-                if (child_errno[i])
-                        continue;
-                pending[i][midx] = hton32 (1);
-                if (idx == -1)
-                        continue;
-                pending[i][idx] = hton32 (1);
-        }
-}
-
 int
 afr_sh_entry_impunge_perform_xattrop (call_frame_t *impunge_frame,
                                       xlator_t *this)
@@ -1120,23 +1041,25 @@ afr_sh_entry_impunge_perform_xattrop (call_frame_t *impunge_frame,
         impunge_sh = &impunge_local->self_heal;
         active_src = impunge_sh->active_source;
 
-        afr_sh_prepare_new_entry_pending_matrix (impunge_local->pending,
-                                                 impunge_sh->child_errno,
-                                                 &impunge_sh->entrybuf,
-                                                 priv->child_count);
+        afr_prepare_new_entry_pending_matrix (impunge_local->pending,
+                                              afr_is_errno_unset,
+                                              impunge_sh->child_errno,
+                                              &impunge_sh->entrybuf,
+                                              priv->child_count);
         xattr = dict_new ();
         if (!xattr) {
                 op_errno = ENOMEM;
                 goto out;
         }
 
-        afr_set_pending_dict (priv, xattr, impunge_local->pending);
+        afr_set_pending_dict (priv, xattr, impunge_local->pending, active_src,
+                              LOCAL_LAST);
 
         STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_xattrop_cbk,
                            (void *) (long) active_src,
                            priv->children[active_src],
                            priv->children[active_src]->fops->xattrop,
-                           &impunge_local->loc, GF_XATTROP_ADD_ARRAY, xattr);
+                           &impunge_local->loc, GF_XATTROP_ADD_ARRAY, xattr, NULL);
 
         if (xattr)
                 dict_unref (xattr);
@@ -1153,7 +1076,7 @@ afr_sh_entry_impunge_newfile_cbk (call_frame_t *impunge_frame, void *cookie,
                                   int32_t op_ret, int32_t op_errno,
                                   inode_t *inode, struct iatt *stbuf,
                                   struct iatt *preparent,
-                                  struct iatt *postparent)
+                                  struct iatt *postparent, dict_t *xdata)
 {
         int              call_count       = 0;
         afr_private_t   *priv             = NULL;
@@ -1198,21 +1121,26 @@ afr_sh_entry_impunge_hardlink_cbk (call_frame_t *impunge_frame, void *cookie,
                                    xlator_t *this, int32_t op_ret,
                                    int32_t op_errno, inode_t *inode,
                                    struct iatt *buf, struct iatt *preparent,
-                                   struct iatt *postparent)
+                                   struct iatt *postparent, dict_t *xdata)
 {
         int              call_count        = 0;
-        call_frame_t    *frame             = NULL;
-        afr_local_t     *impunge_local     = NULL;
-        afr_local_t     *local             = NULL;
-        afr_self_heal_t *impunge_sh        = NULL;
-        afr_self_heal_t *sh                = NULL;
+        afr_local_t     *impunge_local = NULL;
+        afr_self_heal_t *impunge_sh  = NULL;
 
-        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
-                                frame, local, sh);
+        impunge_local = impunge_frame->local;
+        impunge_sh = &impunge_local->self_heal;
+
+        if (IA_IFLNK == impunge_sh->entrybuf.ia_type) {
+                //For symlinks impunge is attempted un-conditionally
+                //So the file can already exist.
+                if ((op_ret < 0) && (op_errno == EEXIST))
+                        op_ret = 0;
+        }
 
         call_count = afr_frame_return (impunge_frame);
         if (call_count == 0)
-                sh->impunge_done (frame, this, op_ret, op_errno);
+                afr_sh_entry_call_impunge_done (impunge_frame, this,
+                                                op_ret, op_errno);
 
         return 0;
 }
@@ -1243,7 +1171,7 @@ afr_sh_entry_impunge_hardlink (call_frame_t *impunge_frame, xlator_t *this,
                            (void *) (long) child_index,
                            priv->children[child_index],
                            priv->children[child_index]->fops->link,
-                           &oldloc, loc);
+                           &oldloc, loc, NULL);
         loc_wipe (&oldloc);
 
         return 0;
@@ -1335,6 +1263,35 @@ afr_sh_entry_impunge_mknod (call_frame_t *impunge_frame, xlator_t *this,
                 gf_log (this->name, GF_LOG_INFO, "%s: gfid set failed",
                         impunge_local->loc.path);
 
+        /*
+         * Reason for adding GLUSTERFS_INTERNAL_FOP_KEY :
+         *
+         * Problem:
+         * While a brick is down in a replica pair, lets say the user creates
+         * one file(file-A) and a hard link to that file(h-file-A). After the
+         * brick comes back up, entry self-heal is attempted on parent dir of
+         * these two files. As part of readdir in self-heal it reads both the
+         * entries file-A and h-file-A for both of them it does name less lookup
+         * to check if there are any hardlinks already present in the
+         * destination brick. It finds that there are no hard links already
+         * present for files file-A, h-file-A. Self-heal does mknods for both
+         * file-A and h-file-A. This leads to file-A and h-file-A not being
+         * hardlinks anymore.
+         *
+         * Fix: (More like shrinking of race-window, the race itself is still
+         * present in posix-mknod).
+         * If mknod comes with the presence of GLUSTERFS_INTERNAL_FOP_KEY then
+         * posix_mknod checks if there are already any gfid-links and does
+         * link() instead of mknod. There still can be a race where two
+         * posix_mknods same gfid see that
+         * gfid-link file is not present and proceeds with mknods and result in
+         * two different files with same gfid.
+         */
+        ret = dict_set_str (dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+        if (ret)
+                gf_log (this->name, GF_LOG_INFO, "%s: %s set failed",
+                        impunge_local->loc.path, GLUSTERFS_INTERNAL_FOP_KEY);
+
         STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_newfile_cbk,
                            (void *) (long) child_index,
                            priv->children[child_index],
@@ -1342,7 +1299,7 @@ afr_sh_entry_impunge_mknod (call_frame_t *impunge_frame, xlator_t *this,
                            &impunge_local->loc,
                            st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type),
                            makedev (ia_major (stbuf->ia_rdev),
-                                    ia_minor (stbuf->ia_rdev)), dict);
+                                    ia_minor (stbuf->ia_rdev)), 0, dict);
 
         if (dict)
                 dict_unref (dict);
@@ -1389,7 +1346,7 @@ afr_sh_entry_impunge_mkdir (call_frame_t *impunge_frame, xlator_t *this,
                            priv->children[child_index]->fops->mkdir,
                            &impunge_local->loc,
                            st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type),
-                           dict);
+                           0, dict);
 
         if (dict)
                 dict_unref (dict);
@@ -1411,7 +1368,7 @@ afr_sh_entry_impunge_symlink (call_frame_t *impunge_frame, xlator_t *this,
         priv = this->private;
         impunge_local = impunge_frame->local;
 
-        buf = &impunge_local->cont.symlink.buf;
+        buf = &impunge_local->cont.dir_fop.buf;
 
         dict = dict_new ();
         if (!dict) {
@@ -1436,7 +1393,7 @@ afr_sh_entry_impunge_symlink (call_frame_t *impunge_frame, xlator_t *this,
                            (void *) (long) child_index,
                            priv->children[child_index],
                            priv->children[child_index]->fops->symlink,
-                           linkname, &impunge_local->loc, dict);
+                           linkname, &impunge_local->loc, 0, dict);
 
         if (dict)
                 dict_unref (dict);
@@ -1450,7 +1407,7 @@ afr_sh_entry_impunge_symlink_unlink_cbk (call_frame_t *impunge_frame,
                                          void *cookie, xlator_t *this,
                                          int32_t op_ret, int32_t op_errno,
                                          struct iatt *preparent,
-                                         struct iatt *postparent)
+                                         struct iatt *postparent, dict_t *xdata)
 {
         afr_private_t   *priv = NULL;
         afr_local_t     *impunge_local = NULL;
@@ -1511,7 +1468,7 @@ afr_sh_entry_impunge_symlink_unlink (call_frame_t *impunge_frame, xlator_t *this
                            (void *) (long) child_index,
                            priv->children[child_index],
                            priv->children[child_index]->fops->unlink,
-                           &impunge_local->loc);
+                           &impunge_local->loc, 0, NULL);
 
         return 0;
 }
@@ -1521,7 +1478,7 @@ int
 afr_sh_entry_impunge_readlink_sink_cbk (call_frame_t *impunge_frame, void *cookie,
                                         xlator_t *this,
                                         int32_t op_ret, int32_t op_errno,
-                                        const char *linkname, struct iatt *sbuf)
+                                        const char *linkname, struct iatt *sbuf, dict_t *xdata)
 {
         afr_private_t   *priv = NULL;
         afr_local_t     *impunge_local = NULL;
@@ -1603,7 +1560,7 @@ afr_sh_entry_impunge_readlink_sink (call_frame_t *impunge_frame, xlator_t *this,
                            (void *) (long) child_index,
                            priv->children[child_index],
                            priv->children[child_index]->fops->readlink,
-                           &impunge_local->loc, 4096);
+                           &impunge_local->loc, 4096, NULL);
 
         return 0;
 }
@@ -1613,7 +1570,7 @@ int
 afr_sh_entry_impunge_readlink_cbk (call_frame_t *impunge_frame, void *cookie,
                                    xlator_t *this,
                                    int32_t op_ret, int32_t op_errno,
-                                   const char *linkname, struct iatt *sbuf)
+                                   const char *linkname, struct iatt *sbuf, dict_t *xdata)
 {
         afr_private_t   *priv = NULL;
         afr_local_t     *impunge_local = NULL;
@@ -1671,13 +1628,13 @@ afr_sh_entry_impunge_readlink (call_frame_t *impunge_frame, xlator_t *this,
         impunge_local = impunge_frame->local;
         impunge_sh = &impunge_local->self_heal;
         active_src = impunge_sh->active_source;
-        impunge_local->cont.symlink.buf = *stbuf;
+        impunge_local->cont.dir_fop.buf = *stbuf;
 
         STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_readlink_cbk,
                            (void *) (long) child_index,
                            priv->children[active_src],
                            priv->children[active_src]->fops->readlink,
-                           &impunge_local->loc, 4096);
+                           &impunge_local->loc, 4096, NULL);
 
         return 0;
 }
@@ -1919,7 +1876,8 @@ afr_sh_entry_common_lookup_done (call_frame_t *impunge_frame, xlator_t *this,
                 afr_sh_common_lookup (impunge_frame, this, &impunge_local->loc,
                                       afr_sh_entry_common_lookup_done, gfid,
                                       AFR_LOOKUP_FAIL_CONFLICTS |
-                                      AFR_LOOKUP_FAIL_MISSING_GFIDS);
+                                      AFR_LOOKUP_FAIL_MISSING_GFIDS,
+                                      NULL);
         } else {
                 afr_sh_entry_call_impunge_recreate (impunge_frame, this);
         }
@@ -1950,14 +1908,7 @@ afr_sh_entry_impunge_entry (call_frame_t *frame, xlator_t *this,
         active_src = sh->active_source;
         sh->impunge_done = afr_sh_entry_impunge_entry_done;
 
-        if ((strcmp (entry->d_name, ".") == 0)
-            || (strcmp (entry->d_name, "..") == 0)
-            || ((strcmp (local->loc.path, "/") == 0)
-                && (strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR) == 0))) {
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "skipping inspection of %s under %s",
-                        entry->d_name, local->loc.path);
+        if (can_skip_entry_self_heal (entry->d_name, &local->loc)) {
                 op_ret = 0;
                 goto out;
         }
@@ -1985,7 +1936,7 @@ afr_sh_entry_impunge_entry (call_frame_t *frame, xlator_t *this,
 
         afr_sh_common_lookup (impunge_frame, this, &impunge_local->loc,
                               afr_sh_entry_common_lookup_done, NULL,
-                              AFR_LOOKUP_FAIL_CONFLICTS);
+                              AFR_LOOKUP_FAIL_CONFLICTS, NULL);
 
         op_ret = 0;
 out:
@@ -2003,7 +1954,7 @@ int
 afr_sh_entry_impunge_readdir_cbk (call_frame_t *frame, void *cookie,
                                   xlator_t *this,
                                   int32_t op_ret, int32_t op_errno,
-                                  gf_dirent_t *entries)
+                                  gf_dirent_t *entries, dict_t *xdata)
 {
         afr_private_t   *priv = NULL;
         afr_local_t     *local  = NULL;
@@ -2026,7 +1977,7 @@ afr_sh_entry_impunge_readdir_cbk (call_frame_t *frame, void *cookie,
                                 local->loc.path,
                                 priv->children[active_src]->name,
                                 strerror (op_errno));
-                        sh->op_failed = 1;
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 } else {
                         gf_log (this->name, GF_LOG_TRACE,
                                 "readdir of %s on subvolume %s complete",
@@ -2076,7 +2027,7 @@ afr_sh_entry_impunge_subvol (call_frame_t *frame, xlator_t *this)
         STACK_WIND (frame, afr_sh_entry_impunge_readdir_cbk,
                     priv->children[active_src],
                     priv->children[active_src]->fops->readdirp,
-                    sh->healing_fd, sh->block_size, sh->offset, 0);
+                    sh->healing_fd, sh->block_size, sh->offset, NULL);
 
         return 0;
 }
@@ -2099,7 +2050,7 @@ afr_sh_entry_impunge_all (call_frame_t *frame, xlator_t *this)
         active_src = next_active_source (frame, this, sh->active_source);
         sh->active_source = active_src;
 
-        if (sh->op_failed) {
+        if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
                 afr_sh_entry_finish (frame, this);
                 return 0;
         }
@@ -2122,7 +2073,7 @@ afr_sh_entry_impunge_all (call_frame_t *frame, xlator_t *this)
 
 int
 afr_sh_entry_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, fd_t *fd)
+                          int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         afr_local_t     *local = NULL;
         afr_self_heal_t *sh = NULL;
@@ -2148,7 +2099,7 @@ afr_sh_entry_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                 local->loc.path,
                                 priv->children[child_index]->name,
                                 strerror (op_errno));
-                        sh->op_failed = 1;
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 }
         }
         UNLOCK (&frame->lock);
@@ -2156,7 +2107,7 @@ afr_sh_entry_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
-                if (sh->op_failed) {
+                if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
                         afr_sh_entry_finish (frame, this);
                         return 0;
                 }
@@ -2194,7 +2145,7 @@ afr_sh_entry_open (call_frame_t *frame, xlator_t *this)
         source  = local->self_heal.source;
         sources = local->self_heal.sources;
 
-        sh->block_size = 65536; //131072
+        sh->block_size = priv->sh_readdir_size;
         sh->offset = 0;
 
         call_count = sh->active_sinks;
@@ -2216,7 +2167,7 @@ afr_sh_entry_open (call_frame_t *frame, xlator_t *this)
                                    (void *) (long) source,
                                    priv->children[source],
                                    priv->children[source]->fops->opendir,
-                                   &local->loc, fd);
+                                   &local->loc, fd, NULL);
                 call_count--;
         }
 
@@ -2233,7 +2184,7 @@ afr_sh_entry_open (call_frame_t *frame, xlator_t *this)
                                    (void *) (long) i,
                                    priv->children[i],
                                    priv->children[i]->fops->opendir,
-                                   &local->loc, fd);
+                                   &local->loc, fd, NULL);
 
                 if (!--call_count)
                         break;
@@ -2288,6 +2239,8 @@ afr_sh_entry_sync_prepare (call_frame_t *frame, xlator_t *this)
                         "merging all entries as a conservative decision",
                         local->loc.path);
 
+        sh->actual_sh_started = _gf_true;
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_SYNC_BEGIN);
         afr_sh_entry_open (frame, this);
 
         return 0;
@@ -2310,7 +2263,7 @@ afr_sh_entry_fix (call_frame_t *frame, xlator_t *this,
         priv = this->private;
 
         if (op_ret < 0) {
-                sh->op_failed = 1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 afr_sh_set_error (sh, op_errno);
                 afr_sh_entry_finish (frame, this);
                 goto out;
@@ -2373,7 +2326,7 @@ afr_sh_post_nonblocking_entry_cbk (call_frame_t *frame, xlator_t *this)
         if (int_lock->lock_op_ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR, "Non Blocking entrylks "
                         "failed for %s.", local->loc.path);
-                sh->op_failed = 1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 afr_sh_entry_done (frame, this);
         } else {
 
@@ -2382,7 +2335,8 @@ afr_sh_post_nonblocking_entry_cbk (call_frame_t *frame, xlator_t *this)
                 afr_sh_common_lookup (frame, this, &local->loc,
                                       afr_sh_entry_fix, NULL,
                                       AFR_LOOKUP_FAIL_CONFLICTS |
-                                      AFR_LOOKUP_FAIL_MISSING_GFIDS);
+                                      AFR_LOOKUP_FAIL_MISSING_GFIDS,
+                                      NULL);
         }
 
         return 0;
@@ -2391,14 +2345,18 @@ afr_sh_post_nonblocking_entry_cbk (call_frame_t *frame, xlator_t *this)
 int
 afr_self_heal_entry (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t   *local = NULL;
+        afr_local_t     *local = NULL;
         afr_private_t   *priv = NULL;
-
+        afr_self_heal_t *sh = NULL;
 
         priv = this->private;
         local = frame->local;
+        sh = &local->self_heal;
+
+        sh->sh_type_in_action = AFR_SELF_HEAL_ENTRY;
 
         if (local->self_heal.do_entry_self_heal && priv->entry_self_heal) {
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_STARTED);
                 afr_sh_entrylk (frame, this, &local->loc, NULL,
                                 afr_sh_post_nonblocking_entry_cbk);
         } else {
diff --git a/xlators/cluster/afr/src/afr-self-heal-metadata.c b/xlators/cluster/afr/src/afr-self-heal-metadata.c
index d0bf382a4..fd5da6cfd 100644
--- a/xlators/cluster/afr/src/afr-self-heal-metadata.c
+++ b/xlators/cluster/afr/src/afr-self-heal-metadata.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -59,20 +50,12 @@ afr_sh_metadata_done (call_frame_t *frame, xlator_t *this)
         sh = &local->self_heal;
 
         afr_sh_reset (frame, this);
-        if (local->govinda_gOvinda) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "split-brain detected, aborting selfheal of %s",
+        if (IA_ISDIR (sh->type)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "proceeding to entry check on %s",
                         local->loc.path);
-                sh->op_failed = 1;
-                sh->completion_cbk (frame, this);
+                afr_self_heal_entry (frame, this);
         } else {
-                if (IA_ISDIR (sh->type)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "proceeding to entry check on %s",
-                                local->loc.path);
-                        afr_self_heal_entry (frame, this);
-                        return 0;
-                }
                 gf_log (this->name, GF_LOG_DEBUG,
                         "proceeding to data check on %s",
                         local->loc.path);
@@ -105,11 +88,24 @@ afr_sh_metadata_finish (call_frame_t *frame, xlator_t *this)
         return 0;
 }
 
+int
+afr_sh_metadata_fail (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t         *local    = NULL;
+        afr_self_heal_t     *sh       = NULL;
+
+        local    = frame->local;
+        sh       = &local->self_heal;
+
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+        afr_sh_metadata_finish (frame, this);
+        return 0;
+}
 
 int
 afr_sh_metadata_erase_pending_cbk (call_frame_t *frame, void *cookie,
                                    xlator_t *this, int32_t op_ret,
-                                   int32_t op_errno, dict_t *xattr)
+                                   int32_t op_errno, dict_t *xattr, dict_t *xdata)
 {
         afr_local_t     *local     = NULL;
         int             call_count = 0;
@@ -141,85 +137,19 @@ afr_sh_metadata_erase_pending_cbk (call_frame_t *frame, void *cookie,
         return 0;
 }
 
-
 int
 afr_sh_metadata_erase_pending (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              i = 0;
-        dict_t          **erase_xattr = NULL;
-
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        afr_sh_pending_to_delta (priv, sh->xattr, sh->delta_matrix,
-                                 sh->success, priv->child_count,
-                                 AFR_METADATA_TRANSACTION);
-
-        erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count,
-                                 gf_afr_mt_dict_t);
-        if (!erase_xattr)
-                return -ENOMEM;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->xattr[i]) {
-                        call_count++;
-
-                        erase_xattr[i] = get_new_dict();
-                        dict_ref (erase_xattr[i]);
-                }
-        }
-
-        afr_sh_delta_to_xattr (priv, sh->delta_matrix, erase_xattr,
-                               priv->child_count, AFR_METADATA_TRANSACTION);
-
-        local->call_count = call_count;
-
-        if (call_count == 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "metadata of %s not healed on any subvolume",
-                        local->loc.path);
-
-                afr_sh_metadata_finish (frame, this);
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!erase_xattr[i])
-                        continue;
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "erasing pending flags from %s on %s",
-                        local->loc.path, priv->children[i]->name);
-
-                STACK_WIND_COOKIE (frame, afr_sh_metadata_erase_pending_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->xattrop,
-                                   &local->loc,
-                                   GF_XATTROP_ADD_ARRAY, erase_xattr[i]);
-                if (!--call_count)
-                        break;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (erase_xattr[i]) {
-                        dict_unref (erase_xattr[i]);
-                }
-        }
-        GF_FREE (erase_xattr);
-
-        return 0;
+         afr_sh_erase_pending (frame, this, AFR_METADATA_TRANSACTION,
+                               afr_sh_metadata_erase_pending_cbk,
+                               afr_sh_metadata_finish);
+         return 0;
 }
 
 
 int
 afr_sh_metadata_sync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t     *local = NULL;
         afr_self_heal_t *sh = NULL;
@@ -250,8 +180,13 @@ afr_sh_metadata_sync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         call_count = afr_frame_return (frame);
 
-        if (call_count == 0)
+        if (call_count == 0) {
+                if (local->xattr_req) {
+                        dict_unref (local->xattr_req);
+                        local->xattr_req = NULL;
+                }
                 afr_sh_metadata_erase_pending (frame, this);
+        }
 
         return 0;
 }
@@ -260,9 +195,9 @@ afr_sh_metadata_sync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 afr_sh_metadata_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                              int32_t op_ret, int32_t op_errno,
-                             struct iatt *preop, struct iatt *postop)
+                             struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
-        afr_sh_metadata_sync_cbk (frame, cookie, this, op_ret, op_errno);
+        afr_sh_metadata_sync_cbk (frame, cookie, this, op_ret, op_errno, xdata);
 
         return 0;
 }
@@ -270,13 +205,93 @@ afr_sh_metadata_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 afr_sh_metadata_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno)
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_sh_metadata_sync_cbk (frame, cookie, this, op_ret, op_errno);
+        afr_sh_metadata_sync_cbk (frame, cookie, this, op_ret, op_errno, xdata);
 
         return 0;
 }
 
+int
+afr_sh_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno,
+                        dict_t *xdata)
+{
+        int            i     = 0;
+        afr_private_t *priv  = NULL;
+        afr_local_t   *local = NULL;
+
+        priv = this->private;
+        local = frame->local;
+
+        if (op_ret < 0) {
+                afr_sh_metadata_sync_cbk (frame, cookie,
+                                          this, -1, op_errno, xdata);
+                goto out;
+        }
+
+        i = (long) cookie;
+
+        STACK_WIND_COOKIE (frame, afr_sh_metadata_xattr_cbk,
+                           (void *) (long) i,
+                           priv->children[i],
+                           priv->children[i]->fops->setxattr,
+                           &local->loc, local->xattr_req, 0, NULL);
+
+ out:
+        return 0;
+}
+
+inline void
+afr_prune_special_keys (dict_t *xattr_dict)
+{
+        dict_del (xattr_dict, GF_SELINUX_XATTR_KEY);
+}
+
+inline void
+afr_prune_pending_keys (dict_t *xattr_dict, afr_private_t *priv)
+{
+        int i = 0;
+
+        for (; i < priv->child_count; i++) {
+                dict_del (xattr_dict, priv->pending_key[i]);
+        }
+}
+
+int
+afr_sh_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                     dict_t *xdata)
+{
+        int            i     = 0;
+        afr_private_t *priv  = NULL;
+        afr_local_t   *local = NULL;
+
+        priv = this->private;
+        local = frame->local;
+
+        if (op_ret < 0) {
+                afr_sh_metadata_sync_cbk (frame, cookie,
+                                          this, -1, op_errno, xdata);
+                goto out;
+        }
+
+        afr_prune_pending_keys (xattr, priv);
+
+        afr_prune_special_keys (xattr);
+
+        i = (long) cookie;
+
+        /* send removexattr in bulk via xdata */
+        STACK_WIND_COOKIE (frame, afr_sh_removexattr_cbk,
+                           cookie,
+                           priv->children[i],
+                           priv->children[i]->fops->removexattr,
+                           &local->loc, "", xattr);
+
+ out:
+        return 0;
+}
 
 int
 afr_sh_metadata_sync (call_frame_t *frame, xlator_t *this, dict_t *xattr)
@@ -302,9 +317,10 @@ afr_sh_metadata_sync (call_frame_t *frame, xlator_t *this, dict_t *xattr)
         /*
          * 2 calls per sink - setattr, setxattr
          */
-        if (xattr)
+        if (xattr) {
                 call_count = active_sinks * 2;
-        else
+                local->xattr_req = dict_ref (xattr);
+        } else
                 call_count = active_sinks;
 
         local->call_count = call_count;
@@ -340,18 +356,18 @@ afr_sh_metadata_sync (call_frame_t *frame, xlator_t *this, dict_t *xattr)
                                    (void *) (long) i,
                                    priv->children[i],
                                    priv->children[i]->fops->setattr,
-                                   &local->loc, &stbuf, valid);
+                                   &local->loc, &stbuf, valid, NULL);
 
                 call_count--;
 
                 if (!xattr)
                         continue;
 
-                STACK_WIND_COOKIE (frame, afr_sh_metadata_xattr_cbk,
+                STACK_WIND_COOKIE (frame, afr_sh_getxattr_cbk,
                                    (void *) (long) i,
                                    priv->children[i],
-                                   priv->children[i]->fops->setxattr,
-                                   &local->loc, xattr, 0);
+                                   priv->children[i]->fops->getxattr,
+                                   &local->loc, NULL, NULL);
                 call_count--;
         }
 
@@ -360,17 +376,15 @@ afr_sh_metadata_sync (call_frame_t *frame, xlator_t *this, dict_t *xattr)
 
 
 int
-afr_sh_metadata_getxattr_cbk (call_frame_t *frame, void *cookie,
-                              xlator_t *this,
-                              int32_t op_ret, int32_t op_errno, dict_t *xattr)
+afr_sh_metadata_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                              dict_t *xdata)
 {
         afr_local_t     *local = NULL;
         afr_self_heal_t *sh = NULL;
         afr_private_t   *priv = NULL;
         int              source = 0;
 
-        int i;
-
         local = frame->local;
         sh = &local->self_heal;
         priv = this->private;
@@ -385,16 +399,147 @@ afr_sh_metadata_getxattr_cbk (call_frame_t *frame, void *cookie,
 
                 afr_sh_metadata_sync (frame, this, NULL);
         } else {
-                for (i = 0; i < priv->child_count; i++) {
-                        dict_del (xattr, priv->pending_key[i]);
-                }
-
+                afr_prune_pending_keys (xattr, priv);
                 afr_sh_metadata_sync (frame, this, xattr);
         }
 
         return 0;
 }
 
+static void
+afr_set_metadata_sh_info_str (afr_local_t *local, afr_self_heal_t *sh,
+                              xlator_t *this)
+{
+        afr_private_t    *priv = NULL;
+        int              i = 0;
+        char             num[1024] = {0};
+        size_t           len = 0;
+        char             *string = NULL;
+        size_t           off = 0;
+        char             *source_child =  " from source %s to";
+        char             *format = " %s, ";
+        char             *string_msg = " metadata self heal";
+        char             *pending_matrix_str = NULL;
+        int              down_child_present = 0;
+        int              unknown_child_present = 0;
+        char             *down_subvol_1 = " down subvolume is ";
+        char             *unknown_subvol_1 = " unknown subvolume is";
+        char             *down_subvol_2 = " down subvolumes are ";
+        char             *unknown_subvol_2 = " unknown subvolumes are ";
+        int              down_count = 0;
+        int              unknown_count = 0;
+
+        priv = this->private;
+
+        pending_matrix_str = afr_get_pending_matrix_str (sh->pending_matrix,
+                                                         this);
+
+        if (!pending_matrix_str)
+                pending_matrix_str = "";
+
+        len += snprintf (num, sizeof (num), "%s", string_msg);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if ((sh->source == i) && (local->child_up[i] == 1)) {
+                        len += snprintf (num, sizeof (num), source_child,
+                                         priv->children[i]->name);
+                } else if ((local->child_up[i] == 1) && (sh->sources[i] == 0)) {
+                        len += snprintf (num, sizeof (num), format,
+                                         priv->children[i]->name);
+                } else if (local->child_up[i] == 0) {
+                        len += snprintf (num, sizeof (num), format,
+                                         priv->children[i]->name);
+                        if (!down_child_present)
+                                down_child_present = 1;
+                        down_count++;
+                } else if (local->child_up[i] == -1) {
+                        len += snprintf (num, sizeof (num), format,
+                                         priv->children[i]->name);
+                        if (!unknown_child_present)
+                                unknown_child_present = 1;
+                        unknown_count++;
+                }
+        }
+
+        if (down_child_present) {
+                if (down_count > 1) {
+                        len += snprintf (num, sizeof (num), "%s",
+                                         down_subvol_2);
+                } else {
+                        len += snprintf (num, sizeof (num), "%s",
+                                         down_subvol_1);
+                }
+        }
+        if (unknown_child_present) {
+                if (unknown_count > 1) {
+                        len += snprintf (num, sizeof (num), "%s",
+                                         unknown_subvol_2);
+                } else {
+                        len += snprintf (num, sizeof (num), "%s",
+                                         unknown_subvol_1);
+                }
+        }
+
+        len ++;
+
+        string = GF_CALLOC (len, sizeof (char), gf_common_mt_char);
+        if (!string)
+                return;
+
+        off += snprintf (string + off, len - off, "%s", string_msg);
+        for (i=0; i < priv->child_count; i++) {
+                if ((sh->source == i) && (local->child_up[i] == 1))
+                        off += snprintf (string + off, len - off, source_child,
+                                         priv->children[i]->name);
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if ((local->child_up[i] == 1)&& (sh->sources[i] == 0))
+                        off += snprintf (string + off, len - off, format,
+                                         priv->children[i]->name);
+        }
+
+        if (down_child_present) {
+                if (down_count > 1) {
+                        off += snprintf (string + off, len - off, "%s",
+                                         down_subvol_2);
+                } else {
+                        off += snprintf (string + off, len - off, "%s",
+                                         down_subvol_1);
+                }
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i] == 0)
+                        off += snprintf (string + off, len - off, format,
+                                         priv->children[i]->name);
+        }
+
+        if (unknown_child_present) {
+                if (unknown_count > 1) {
+                        off += snprintf (string + off, len - off, "%s",
+                                 unknown_subvol_2);
+                } else {
+                        off += snprintf (string + off, len - off, "%s",
+                                         unknown_subvol_1);
+                }
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i] == -1)
+                        off += snprintf (string + off, len - off, format,
+                                         priv->children[i]->name);
+        }
+
+        gf_asprintf (&sh->metadata_sh_info, "%s metadata %s,", string,
+                     pending_matrix_str);
+
+        if (pending_matrix_str && strcmp (pending_matrix_str, ""))
+                GF_FREE (pending_matrix_str);
+
+        if (string && strcmp (string, ""))
+                GF_FREE (string);
+}
 
 int
 afr_sh_metadata_sync_prepare (call_frame_t *frame, xlator_t *this)
@@ -424,10 +569,13 @@ afr_sh_metadata_sync_prepare (call_frame_t *frame, xlator_t *this)
                 local->loc.path, priv->children[source]->name,
                 sh->active_sinks);
 
+        sh->actual_sh_started = _gf_true;
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_SYNC_BEGIN);
+        afr_set_metadata_sh_info_str (local, sh, this);
         STACK_WIND (frame, afr_sh_metadata_getxattr_cbk,
                     priv->children[source],
                     priv->children[source]->fops->getxattr,
-                    &local->loc, NULL);
+                    &local->loc, NULL, NULL);
 
         return 0;
 }
@@ -449,7 +597,7 @@ afr_sh_metadata_fix (call_frame_t *frame, xlator_t *this,
         priv = this->private;
 
         if (op_ret < 0) {
-                sh->op_failed = 1;
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
                 afr_sh_set_error (sh, op_errno);
                 afr_sh_metadata_finish (frame, this);
                 goto out;
@@ -458,15 +606,6 @@ afr_sh_metadata_fix (call_frame_t *frame, xlator_t *this,
                                       sh->pending_matrix, sh->sources,
                                       sh->success_children,
                                       AFR_METADATA_TRANSACTION, NULL, _gf_false);
-        if (nsources == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No self-heal needed for %s",
-                        local->loc.path);
-
-                afr_sh_metadata_finish (frame, this);
-                goto out;
-        }
-
         if ((nsources == -1)
             && (priv->favorite_child != -1)
             && (sh->child_errno[priv->favorite_child] == 0)) {
@@ -483,12 +622,18 @@ afr_sh_metadata_fix (call_frame_t *frame, xlator_t *this,
         }
 
         if (nsources == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to self-heal permissions/ownership of '%s' "
-                        "(possible split-brain). Please fix the file on "
-                        "all backend volumes", local->loc.path);
+                afr_sh_print_split_brain_log (sh->pending_matrix, this,
+                                              local->loc.path);
+                afr_set_split_brain (this, sh->inode, SPB, DONT_KNOW);
+                afr_sh_metadata_fail (frame, this);
+                goto out;
+        }
 
-                local->govinda_gOvinda = 1;
+        afr_set_split_brain (this, sh->inode, NO_SPB, DONT_KNOW);
+        if (nsources == 0) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "No self-heal needed for %s",
+                        local->loc.path);
 
                 afr_sh_metadata_finish (frame, this);
                 goto out;
@@ -527,7 +672,10 @@ afr_sh_metadata_fix (call_frame_t *frame, xlator_t *this,
                                         sh->fresh_children);
         }
 
-        afr_sh_metadata_sync_prepare (frame, this);
+        if (sh->do_metadata_self_heal && priv->metadata_self_heal)
+                afr_sh_metadata_sync_prepare (frame, this);
+        else
+                afr_sh_metadata_finish (frame, this);
 out:
         return;
 }
@@ -543,9 +691,9 @@ afr_sh_metadata_post_nonblocking_inodelk_cbk (call_frame_t *frame,
         int_lock = &local->internal_lock;
 
         if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Non Blocking metadata "
+                gf_log (this->name, GF_LOG_DEBUG, "Non Blocking metadata "
                         "inodelks failed for %s.", local->loc.path);
-                gf_log (this->name, GF_LOG_ERROR, "Metadata self-heal "
+                gf_log (this->name, GF_LOG_DEBUG, "Metadata self-heal "
                         "failed for %s.", local->loc.path);
                 afr_sh_metadata_done (frame, this);
         } else {
@@ -556,7 +704,8 @@ afr_sh_metadata_post_nonblocking_inodelk_cbk (call_frame_t *frame,
                 afr_sh_common_lookup (frame, this, &local->loc,
                                       afr_sh_metadata_fix, NULL,
                                       AFR_LOOKUP_FAIL_CONFLICTS |
-                                      AFR_LOOKUP_FAIL_MISSING_GFIDS);
+                                      AFR_LOOKUP_FAIL_MISSING_GFIDS,
+                                      NULL);
         }
 
         return 0;
@@ -566,19 +715,22 @@ int
 afr_sh_metadata_lock (call_frame_t *frame, xlator_t *this)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
 
+        int_lock->domain           = this->name;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
         int_lock->transaction_lk_type = AFR_SELFHEAL_LK;
         int_lock->selfheal_lk_type    = AFR_METADATA_SELF_HEAL_LK;
 
         afr_set_lock_number (frame, this);
 
-        int_lock->lk_flock.l_start = LLONG_MAX - 1;
-        int_lock->lk_flock.l_len   = 0;
-        int_lock->lk_flock.l_type  = F_WRLCK;
+        inodelk->flock.l_start = LLONG_MAX - 1;
+        inodelk->flock.l_len   = 0;
+        inodelk->flock.l_type  = F_WRLCK;
         int_lock->lock_cbk         = afr_sh_metadata_post_nonblocking_inodelk_cbk;
 
         afr_nonblocking_inodelk (frame, this);
@@ -586,17 +738,29 @@ afr_sh_metadata_lock (call_frame_t *frame, xlator_t *this)
         return 0;
 }
 
+gf_boolean_t
+afr_can_start_metadata_self_heal (afr_self_heal_t *sh, afr_private_t *priv)
+{
+        if (sh->force_confirm_spb)
+                return _gf_true;
+        if (sh->do_metadata_self_heal && priv->metadata_self_heal)
+                return _gf_true;
+        return _gf_false;
+}
 
 int
 afr_self_heal_metadata (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t   *local = NULL;
         afr_private_t *priv = this->private;
-
+        afr_self_heal_t *sh = &local->self_heal;
 
         local = frame->local;
+        sh = &local->self_heal;
+        sh->sh_type_in_action = AFR_SELF_HEAL_METADATA;
 
-        if (local->self_heal.do_metadata_self_heal && priv->metadata_self_heal) {
+        if (afr_can_start_metadata_self_heal (sh, priv)) {
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_STARTED);
                 afr_sh_metadata_lock (frame, this);
         } else {
                 afr_sh_metadata_done (frame, this);
diff --git a/xlators/cluster/afr/src/afr-self-heal.h b/xlators/cluster/afr/src/afr-self-heal.h
index f40c06faa..7c9bc8111 100644
--- a/xlators/cluster/afr/src/afr-self-heal.h
+++ b/xlators/cluster/afr/src/afr-self-heal.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __AFR_SELF_HEAL_H__
@@ -30,13 +21,6 @@
 #define SIZE_GREATER(buf1,buf2) ((buf1)->ia_size > (buf2)->ia_size)
 
 int
-afr_sh_has_metadata_pending (dict_t *xattr, xlator_t *this);
-int
-afr_sh_has_entry_pending (dict_t *xattr, xlator_t *this);
-int
-afr_sh_has_data_pending (dict_t *xattr, xlator_t *this);
-
-int
 afr_self_heal_entry (call_frame_t *frame, xlator_t *this);
 
 int
@@ -54,5 +38,6 @@ afr_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode);
 int
 afr_lookup_select_read_child_by_txn_type (xlator_t *this, afr_local_t *local,
                                           dict_t **xattr,
-                                          afr_transaction_type txn_type);
+                                          afr_transaction_type txn_type,
+                                          uuid_t gfid);
 #endif /* __AFR_SELF_HEAL_H__ */
diff --git a/xlators/cluster/afr/src/afr-self-heald.c b/xlators/cluster/afr/src/afr-self-heald.c
index 3068d5c46..1b48a1bca 100644
--- a/xlators/cluster/afr/src/afr-self-heald.c
+++ b/xlators/cluster/afr/src/afr-self-heald.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -28,20 +19,18 @@
 #include "protocol-common.h"
 #include "event-history.h"
 
-#define AFR_POLL_TIMEOUT 600
-
 typedef enum {
         STOP_CRAWL_ON_SINGLE_SUBVOL = 1
 } afr_crawl_flags_t;
 
 typedef enum {
         HEAL = 1,
-        INFO
+        INFO,
+        STATISTICS_TO_BE_HEALED,
 } shd_crawl_op;
 
 typedef struct shd_dump {
         dict_t   *dict;
-        time_t   sh_time;
         xlator_t *this;
         int      child;
 } shd_dump_t;
@@ -69,9 +58,59 @@ afr_start_crawl (xlator_t *this, int idx, afr_crawl_type_t crawl,
 static int
 _crawl_directory (fd_t *fd, loc_t *loc, afr_crawl_data_t *crawl_data);
 
+/* For calling straight through (e.g. already in a synctask). */
+int
+afr_find_child_position (xlator_t *this, int child, afr_child_pos_t *pos);
+
+/* For deferring through a new synctask. */
 int
 afr_syncop_find_child_position (void *data);
 
+static int
+_loc_assign_gfid_path (loc_t *loc)
+{
+        int  ret = -1;
+        char gfid_path[64] = {0};
+
+        if (loc->inode && !uuid_is_null (loc->inode->gfid)) {
+                ret = inode_path (loc->inode, NULL, (char**)&loc->path);
+        } else if (!uuid_is_null (loc->gfid)) {
+                snprintf (gfid_path, sizeof (gfid_path), "<gfid:%s>",
+                          uuid_utoa (loc->gfid));
+                loc->path = gf_strdup (gfid_path);
+                if (loc->path)
+                        ret = 0;
+        }
+        return ret;
+}
+
+void
+_destroy_crawl_event_data (void *data)
+{
+        shd_crawl_event_t        *crawl_event = NULL;
+
+        if (!data)
+                goto out;
+
+        crawl_event = (shd_crawl_event_t *)data;
+        GF_FREE (crawl_event->start_time_str);
+        GF_FREE (crawl_event->end_time_str);
+
+out:
+        return;
+}
+
+void
+_destroy_shd_event_data (void *data)
+{
+        shd_event_t             *event = NULL;
+        if (!data)
+                goto out;
+        event = (shd_event_t*)data;
+        GF_FREE (event->path);
+out:
+        return;
+}
 void
 shd_cleanup_event (void *event)
 {
@@ -79,8 +118,7 @@ shd_cleanup_event (void *event)
 
         if (!shd_event)
                 goto out;
-        if (shd_event->path)
-                GF_FREE (shd_event->path);
+        GF_FREE (shd_event->path);
         GF_FREE (shd_event);
 out:
         return;
@@ -118,8 +156,125 @@ _build_index_loc (xlator_t *this, loc_t *loc, char *name, loc_t *parent)
 }
 
 int
-_add_str_to_dict (xlator_t *this, dict_t *output, int child, char *str,
-                  gf_boolean_t dyn)
+_add_crawl_stats_to_dict (xlator_t *this, dict_t *output, int child,
+                          shd_crawl_event_t *shd_event, struct timeval *tv)
+{
+        int             ret = 0;
+        uint64_t        count = 0;
+        char            key[256] = {0};
+        int             xl_id = 0;
+        uint64_t        healed_count = 0;
+        uint64_t        split_brain_count = 0;
+        uint64_t        heal_failed_count = 0;
+        char            *start_time_str = NULL;
+        char            *end_time_str = NULL;
+        char            *crawl_type = NULL;
+        int             progress = -1;
+
+        healed_count = shd_event->healed_count;
+        split_brain_count = shd_event->split_brain_count;
+        heal_failed_count = shd_event->heal_failed_count;
+        start_time_str = shd_event->start_time_str;
+        end_time_str = shd_event->end_time_str;
+        crawl_type = shd_event->crawl_type;
+
+        if (!start_time_str) {
+                ret = -1;
+                goto out;
+        }
+
+
+        ret = dict_get_int32 (output, this->name, &xl_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "xl does not have id");
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "statistics-%d-%d-count", xl_id, child);
+        ret = dict_get_uint64 (output, key, &count);
+
+        snprintf (key, sizeof (key), "statistics_healed_cnt-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_uint64(output, key, healed_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "healed_count to outout");
+                goto out;
+         }
+        snprintf (key, sizeof (key), "statistics_sb_cnt-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_uint64 (output, key, split_brain_count);
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "split_brain_count to outout");
+                goto out;
+        }
+        snprintf (key, sizeof (key), "statistics_crawl_type-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_dynstr (output, key, gf_strdup (crawl_type));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "crawl_type to output");
+                goto out;
+        }
+        snprintf (key, sizeof (key), "statistics_heal_failed_cnt-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_uint64 (output, key, heal_failed_count);
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "healed_failed_count to outout");
+                goto out;
+        }
+        snprintf (key, sizeof (key), "statistics_strt_time-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_dynstr (output, key, gf_strdup(start_time_str));
+
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "crawl_start_time to outout");
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "statistics_end_time-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+
+        if (!end_time_str)
+                end_time_str = "Could not determine the end time";
+        ret = dict_set_dynstr (output, key, gf_strdup(end_time_str));
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "crawl_end_time to outout");
+                goto out;
+        }
+        snprintf (key, sizeof (key), "statistics_inprogress-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+
+        if (shd_event->crawl_inprogress == _gf_true)
+                progress = 1;
+        else
+                progress = 0;
+
+        ret = dict_set_int32 (output, key, progress);
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "inprogress to outout");
+                goto out;
+        }
+
+         snprintf (key, sizeof (key), "statistics-%d-%d-count",xl_id, child);
+         ret = dict_set_uint64 (output, key, count + 1);
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not increment the "
+                        "counter.");
+                goto out;
+         }
+out:
+        return ret;
+}
+
+int
+_add_path_to_dict (xlator_t *this, dict_t *output, int child, char *path,
+                   struct timeval *tv, gf_boolean_t dyn)
 {
         //subkey not used for now
         int             ret = -1;
@@ -138,15 +293,27 @@ _add_str_to_dict (xlator_t *this, dict_t *output, int child, char *str,
 
         snprintf (key, sizeof (key), "%d-%d-%"PRIu64, xl_id, child, count);
         if (dyn)
-                ret = dict_set_dynstr (output, key, str);
+                ret = dict_set_dynstr (output, key, path);
         else
-                ret = dict_set_str (output, key, str);
+                ret = dict_set_str (output, key, path);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "%s: Could not add to output",
-                        str);
+                        path);
                 goto out;
         }
 
+        if (!tv)
+                goto inc_count;
+        snprintf (key, sizeof (key), "%d-%d-%"PRIu64"-time", xl_id,
+                  child, count);
+        ret = dict_set_uint32 (output, key, tv->tv_sec);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s: Could not set time",
+                        path);
+                goto out;
+        }
+
+inc_count:
         snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
         ret = dict_set_uint64 (output, key, count + 1);
         if (ret) {
@@ -160,19 +327,21 @@ out:
 
 int
 _get_path_from_gfid_loc (xlator_t *this, xlator_t *readdir_xl, loc_t *child,
-                         char **fpath)
+                         char **fpath, gf_boolean_t *missing)
 {
         dict_t          *xattr = NULL;
         char            *path = NULL;
         int             ret = -1;
 
-        ret = syncop_getxattr (readdir_xl, child, &xattr,
-                               GFID_TO_PATH_KEY);
-        if (ret)
+        ret = syncop_getxattr (readdir_xl, child, &xattr, GFID_TO_PATH_KEY);
+        if (ret < 0) {
+                if ((errno == ENOENT) && missing)
+                        *missing = _gf_true;
                 goto out;
+        }
         ret = dict_get_str (xattr, GFID_TO_PATH_KEY, &path);
         if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "Failed to get path for "
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get path for "
                         "gfid %s", uuid_utoa (child->gfid));
                 goto out;
         }
@@ -201,54 +370,57 @@ _add_event_to_dict (circular_buffer_t *cb, void *data)
         shd_event = cb->data;
         if (shd_event->child != dump_data->child)
                 goto out;
-        if (cb->tv.tv_sec >= dump_data->sh_time)
-                ret = _add_str_to_dict (dump_data->this, dump_data->dict,
-                                        dump_data->child, shd_event->path,
-                                        _gf_false);
+        ret = _add_path_to_dict (dump_data->this, dump_data->dict,
+                                 dump_data->child, shd_event->path, &cb->tv,
+                                 _gf_false);
 out:
         return ret;
 }
 
 int
-_add_eh_to_dict (xlator_t *this, eh_t *eh, dict_t *dict, time_t sh_time,
-                 int child)
+_add_crawl_event_statistics_to_dict (circular_buffer_t *cb, void *data)
+{
+        int               ret = 0;
+        shd_dump_t        *dump_data = NULL;
+        shd_crawl_event_t *shd_event = NULL;
+
+        dump_data = data;
+        shd_event = cb->data;
+        ret = _add_crawl_stats_to_dict (dump_data->this, dump_data->dict,
+                                        dump_data->child, shd_event, &cb->tv);
+        return ret;
+}
+
+int
+_add_eh_to_dict (xlator_t *this, eh_t *eh, dict_t *dict, int child)
 {
         shd_dump_t dump_data = {0};
 
         dump_data.this = this;
         dump_data.dict = dict;
-        dump_data.sh_time = sh_time;
         dump_data.child = child;
         eh_dump (eh, &dump_data, _add_event_to_dict);
         return 0;
 }
 
+
 int
-_add_summary_to_dict (xlator_t *this, afr_crawl_data_t *crawl_data,
-                      gf_dirent_t *entry,
-                      loc_t *childloc, loc_t *parentloc, struct iatt *iattr)
+_add_statistics_to_dict (xlator_t *this, dict_t *dict, int child)
 {
-        dict_t          *output = NULL;
-        xlator_t        *readdir_xl = NULL;
-        int             ret = -1;
-        char            *path = NULL;
-
-        if (uuid_is_null (childloc->gfid))
-                goto out;
+        shd_dump_t              dump_data = {0};
+        afr_private_t           *priv  = NULL;
+        afr_self_heald_t        *shd = NULL;
 
-        output = crawl_data->op_data;
-        readdir_xl = crawl_data->readdir_xl;
+        priv = this->private;
+        shd = &priv->shd;
 
-        ret = _get_path_from_gfid_loc (this, readdir_xl, childloc, &path);
-        if (ret)
-                goto out;
+        dump_data.this = this;
+        dump_data.dict = dict;
+        dump_data.child = child;
+        eh_dump (shd->statistics[child], &dump_data,
+                 _add_crawl_event_statistics_to_dict);
+        return 0;
 
-        ret = _add_str_to_dict (this, output, crawl_data->child, path,
-                                _gf_true);
-out:
-        if (ret && path)
-                GF_FREE (path);
-        return ret;
 }
 
 void
@@ -261,13 +433,13 @@ _remove_stale_index (xlator_t *this, xlator_t *readdir_xl,
         ret = _build_index_loc (this, &index_loc, fname, parent);
         if (ret)
                 goto out;
-        gf_log (this->name, GF_LOG_INFO, "Removing stale index "
+        gf_log (this->name, GF_LOG_DEBUG, "Removing stale index "
                 "for %s on %s", index_loc.name, readdir_xl->name);
         ret = syncop_unlink (readdir_xl, &index_loc);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s: Failed to remove"
-                        " index on %s - %s", index_loc.name,
-                        readdir_xl->name, strerror (errno));
+        if(ret && (errno != ENOENT)) {
+                gf_log(this->name, GF_LOG_ERROR, "%s: Failed to remove index "
+                       "on %s - %s",index_loc.name, readdir_xl->name,
+                       strerror (errno));
         }
         index_loc.path = NULL;
         loc_wipe (&index_loc);
@@ -275,28 +447,108 @@ out:
         return;
 }
 
+int
+_count_hard_links_under_base_indices_dir (xlator_t *this,
+                                           afr_crawl_data_t *crawl_data,
+                                           gf_dirent_t *entry, loc_t *childloc,
+                                           loc_t *parentloc, struct iatt *iattr)
+{
+        xlator_t                *readdir_xl = crawl_data->readdir_xl;
+        struct iatt             parent = {0};
+        int                     ret = 0;
+        dict_t                  *output = NULL;
+        int                     xl_id =  0;
+        char                    key[256] = {0};
+        int                     child  = -1;
+        uint64_t                     hardlinks = 0;
+
+        output = crawl_data->op_data;
+        child = crawl_data->child;
+
+        ret = syncop_lookup (readdir_xl, childloc, NULL, iattr, NULL, &parent);
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (output, this->name, &xl_id);
+        if (ret)
+                goto out;
+
+        snprintf (key, sizeof (key), "%d-%d-hardlinks", xl_id, child);
+        ret =  dict_get_uint64 (output, key, &hardlinks);
+
+        /*Removing the count of base_entry under indices/base_indicies and
+         * entry under indices/xattrop */
+        hardlinks = hardlinks + iattr->ia_nlink - 2;
+        ret = dict_set_uint64 (output, key, hardlinks);
+        if (ret)
+                goto out;
+
+out:
+        return ret;
+}
+
+int
+_add_summary_to_dict (xlator_t *this, afr_crawl_data_t *crawl_data,
+                      gf_dirent_t *entry,
+                      loc_t *childloc, loc_t *parentloc, struct iatt *iattr)
+{
+        dict_t          *output = NULL;
+        xlator_t        *readdir_xl = NULL;
+        int             ret = -1;
+        char            *path = NULL;
+        gf_boolean_t    missing = _gf_false;
+        char            gfid_str[64] = {0};
+
+        if (uuid_is_null (childloc->gfid))
+                goto out;
+
+        output = crawl_data->op_data;
+        readdir_xl = crawl_data->readdir_xl;
+
+        ret = _get_path_from_gfid_loc (this, readdir_xl, childloc, &path,
+                                       &missing);
+        if (ret == 0) {
+                ret = _add_path_to_dict (this, output, crawl_data->child, path,
+                                         NULL, _gf_true);
+        } else if (missing) {
+                _remove_stale_index (this, readdir_xl, parentloc,
+                                     uuid_utoa_r (childloc->gfid, gfid_str));
+        }
+
+out:
+        if (ret && path)
+                GF_FREE (path);
+        return ret;
+}
+
 void
 _crawl_post_sh_action (xlator_t *this, loc_t *parent, loc_t *child,
-                       int32_t op_ret, int32_t op_errno,
+                       int32_t op_ret, int32_t op_errno, dict_t *xattr_rsp,
                        afr_crawl_data_t *crawl_data)
 {
-        int              ret = 0;
-        afr_private_t    *priv = NULL;
-        afr_self_heald_t *shd = NULL;
-        eh_t             *eh = NULL;
-        char             *path = NULL;
-        shd_event_t      *event = NULL;
+        int                ret = 0;
+        afr_private_t      *priv = NULL;
+        afr_self_heald_t   *shd = NULL;
+        eh_t               *eh = NULL;
+        char               *path = NULL;
+        char               gfid_str[64] = {0};
+        shd_event_t        *event = NULL;
+        int32_t            sh_failed = 0;
+        gf_boolean_t       split_brain = 0;
+        int32_t            actual_sh_done = 0;
+        shd_crawl_event_t  **shd_crawl_event = NULL;
 
         priv = this->private;
         shd  = &priv->shd;
         if (crawl_data->crawl == INDEX) {
                 if ((op_ret < 0) && (op_errno == ENOENT)) {
                         _remove_stale_index (this, crawl_data->readdir_xl,
-                                             parent, uuid_utoa (child->gfid));
+                                             parent, uuid_utoa_r (child->gfid,
+                                                                  gfid_str));
                         goto out;
                 }
                 ret = _get_path_from_gfid_loc (this, crawl_data->readdir_xl,
-                                               child, &path);
+                                               child, &path, NULL);
                 if (ret)
                         goto out;
         } else {
@@ -307,23 +559,45 @@ _crawl_post_sh_action (xlator_t *this, loc_t *parent, loc_t *child,
                 }
         }
 
-        if (op_ret < 0 && op_errno == EIO)
+        if (xattr_rsp) {
+                ret = dict_get_int32 (xattr_rsp, "sh-failed", &sh_failed);
+                ret = dict_get_int32 (xattr_rsp, "actual-sh-done", &actual_sh_done);
+        }
+
+        shd_crawl_event = (shd_crawl_event_t**)(shd->crawl_events);
+
+        split_brain = afr_is_split_brain (this, child->inode);
+        if ((op_ret < 0 && op_errno == EIO) || split_brain) {
                 eh = shd->split_brain;
-        else if (op_ret < 0)
+                shd_crawl_event[crawl_data->child]->split_brain_count += 1;
+        } else if ((op_ret < 0) || sh_failed) {
                 eh = shd->heal_failed;
-        else
+                shd_crawl_event[crawl_data->child]->heal_failed_count += 1;
+        } else if (actual_sh_done == 1) {
                 eh = shd->healed;
+                shd_crawl_event[crawl_data->child]->healed_count += 1;
+        }
         ret = -1;
-        event = GF_CALLOC (1, sizeof (*event), gf_afr_mt_shd_event_t);
-        if (!event)
-                goto out;
-        event->child = crawl_data->child;
-        event->path = path;
-        ret = eh_save_history (eh, event);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "%s:Failed to save to "
-                        "eh, (%d, %s)", path, op_ret, strerror (op_errno));
-                goto out;
+
+        if (eh != NULL) {
+                event = GF_CALLOC (1, sizeof (*event), gf_afr_mt_shd_event_t);
+                if (!event)
+                        goto out;
+                event->child = crawl_data->child;
+                event->path = path;
+
+                ret = eh_save_history (eh, event);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s:Failed to save "
+                                "to event history, (%d, %s)", path, op_ret,
+                                strerror (op_errno));
+
+                        goto out;
+                }
+        } else {
+                gf_log (this->name, GF_LOG_DEBUG, "%s:Self heal already done ",
+                        path);
+
         }
         ret = 0;
 out:
@@ -333,21 +607,56 @@ out:
 }
 
 int
+_link_inode_update_loc (xlator_t *this, loc_t *loc, struct iatt *iattr)
+{
+        inode_t       *link_inode = NULL;
+        int           ret = -1;
+
+        link_inode = inode_link (loc->inode, NULL, NULL, iattr);
+        if (link_inode == NULL) {
+                gf_log (this->name, GF_LOG_ERROR, "inode link failed "
+                        "on the inode (%s)", uuid_utoa (iattr->ia_gfid));
+                goto out;
+        }
+        inode_unref (loc->inode);
+        loc->inode = link_inode;
+        ret = 0;
+out:
+        return ret;
+}
+
+int
 _self_heal_entry (xlator_t *this, afr_crawl_data_t *crawl_data, gf_dirent_t *entry,
                   loc_t *child, loc_t *parent, struct iatt *iattr)
 {
         struct iatt      parentbuf = {0};
         int              ret = 0;
+        dict_t           *xattr_rsp = NULL;
+        dict_t           *xattr_req = NULL;
 
-        if (uuid_is_null (child->gfid))
-                gf_log (this->name, GF_LOG_DEBUG, "lookup %s", child->path);
-        else
-                gf_log (this->name, GF_LOG_DEBUG, "lookup %s",
-                        uuid_utoa (child->gfid));
+        xattr_req = dict_new ();
+        if (!xattr_req) {
+                errno = ENOMEM;
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_int32 (xattr_req, "allow-sh-for-running-transaction", 1);
+
+        gf_log (this->name, GF_LOG_DEBUG, "lookup %s", child->path);
+
+        ret = syncop_lookup (this, child, xattr_req,
+                             iattr, &xattr_rsp, &parentbuf);
+        _crawl_post_sh_action (this, parent, child, ret, errno, xattr_rsp,
+                               crawl_data);
+        if (xattr_rsp)
+                dict_unref (xattr_rsp);
+        if (ret == 0)
+                ret = _link_inode_update_loc (this, child, iattr);
 
-        ret = syncop_lookup (this, child, NULL,
-                             iattr, NULL, &parentbuf);
-        _crawl_post_sh_action (this, parent, child, ret, errno, crawl_data);
+out:
+        if (xattr_req)
+                dict_unref(xattr_req);
         return ret;
 }
 
@@ -362,13 +671,6 @@ afr_crawl_done  (int ret, call_frame_t *sync_frame, void *data)
 void
 _do_self_heal_on_subvol (xlator_t *this, int child, afr_crawl_type_t crawl)
 {
-        afr_private_t   *priv = NULL;
-        afr_self_heald_t *shd = NULL;
-
-        priv = this->private;
-        shd = &priv->shd;
-
-        time (&shd->sh_times[child]);
         afr_start_crawl (this, child, crawl, _self_heal_entry,
                          NULL, _gf_true, STOP_CRAWL_ON_SINGLE_SUBVOL,
                          afr_crawl_done);
@@ -386,11 +688,11 @@ _crawl_proceed (xlator_t *this, int child, int crawl_flags, char **reason)
         shd  = &priv->shd;
         if (!shd->enabled) {
                 msg = "Self-heal daemon is not enabled";
-                gf_log (this->name, GF_LOG_ERROR, msg);
+                gf_log (this->name, GF_LOG_DEBUG, "%s", msg);
                 goto out;
         }
         if (!priv->child_up[child]) {
-                gf_log (this->name, GF_LOG_ERROR, "Stopping crawl for %s , "
+                gf_log (this->name, GF_LOG_DEBUG, "Stopping crawl for %s , "
                         "subvol went down", priv->children[child]->name);
                 msg = "Brick is Not connected";
                 goto out;
@@ -399,7 +701,7 @@ _crawl_proceed (xlator_t *this, int child, int crawl_flags, char **reason)
         if (crawl_flags & STOP_CRAWL_ON_SINGLE_SUBVOL) {
                 if (afr_up_children_count (priv->child_up,
                                            priv->child_count) < 2) {
-                        gf_log (this->name, GF_LOG_ERROR, "Stopping crawl as "
+                        gf_log (this->name, GF_LOG_DEBUG, "Stopping crawl as "
                                 "< 2 children are up");
                         msg = "< 2 bricks in replica are running";
                         goto out;
@@ -431,20 +733,28 @@ _do_crawl_op_on_local_subvols (xlator_t *this, afr_crawl_type_t crawl,
         if (op == HEAL)
                 crawl_flags |= STOP_CRAWL_ON_SINGLE_SUBVOL;
 
-        ret = dict_get_int32 (output, this->name, &xl_id);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Invalid input, "
-                        "translator-id is not available");
-                goto out;
+        if (output) {
+                ret = dict_get_int32 (output, this->name, &xl_id);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Invalid input, "
+                                "translator-id is not available");
+                        goto out;
+                }
         }
         pos_data.this = this;
         subkey = "status";
         for (i = 0; i < priv->child_count; i++) {
                 if (_crawl_proceed (this, i, crawl_flags, &status)) {
                         pos_data.child = i;
-                        ret = synctask_new (this->ctx->env,
-                                            afr_syncop_find_child_position,
-                                            NULL, NULL, &pos_data);
+                        /*
+                         * We're already in a synctask in this case, so we
+                         * don't need to defer through a second (and in fact
+                         * that can cause deadlock).  Just call straight
+                         * through instead.
+                         */
+                        ret = afr_find_child_position(pos_data.this,
+                                                      pos_data.child,
+                                                      &pos_data.pos);
                         if (ret) {
                                 status = "Not able to find brick location";
                         } else if (pos_data.pos == AFR_POS_REMOTE) {
@@ -455,22 +765,35 @@ _do_crawl_op_on_local_subvols (xlator_t *this, afr_crawl_type_t crawl,
                                         status = "Started self-heal";
                                         _do_self_heal_on_subvol (this, i,
                                                                  crawl);
-                                } else {
+                                } else if (output && (op == INFO)) {
                                         status = "";
                                         afr_start_crawl (this, i, INDEX,
                                                          _add_summary_to_dict,
                                                          output, _gf_false, 0,
                                                          NULL);
+                                } else if (output &&
+                                           (op == STATISTICS_TO_BE_HEALED)) {
+                                            status = "";
+                                            afr_start_crawl (this, i,
+                                                             INDEX_TO_BE_HEALED,
+                                       _count_hard_links_under_base_indices_dir,
+                                                             output, _gf_false,
+                                                             0, NULL);
                                 }
                         }
-                        snprintf (key, sizeof (key), "%d-%d-%s", xl_id,
-                                  i, subkey);
-                        ret = dict_set_str (output, key, status);
+                        if (output) {
+                                snprintf (key, sizeof (key), "%d-%d-%s", xl_id,
+                                          i, subkey);
+                                ret = dict_set_str (output, key, status);
+                        }
                         if (!op_ret && (crawl == FULL))
                                 break;
                 }
-                snprintf (key, sizeof (key), "%d-%d-%s", xl_id, i, subkey);
-                ret = dict_set_str (output, key, status);
+                if (output) {
+                        snprintf (key, sizeof (key), "%d-%d-%s", xl_id, i,
+                                  subkey);
+                        ret = dict_set_str (output, key, status);
+                }
         }
 out:
         return op_ret;
@@ -489,8 +812,103 @@ _get_index_summary_on_local_subvols (xlator_t *this, dict_t *output)
         return _do_crawl_op_on_local_subvols (this, INDEX, INFO, output);
 }
 
+void
+afr_fill_completed_crawl_statistics_to_dict (xlator_t *this, dict_t *dict)
+{
+        afr_private_t           *priv  = NULL;
+        afr_self_heald_t        *shd = NULL;
+        int                     i = 0;
+        priv = this->private;
+        shd= &priv->shd;
+        for (i = 0; i < priv->child_count; i++) {
+                if (shd->pos[i] != AFR_POS_LOCAL)
+                        continue;
+                _add_statistics_to_dict (this, dict, i);
+        }
+
+        return ;
+}
+
+static void
+reset_crawl_event (shd_crawl_event_t *crawl_event)
+{
+    crawl_event->healed_count = 0;
+    crawl_event->split_brain_count = 0;
+    crawl_event->heal_failed_count = 0;
+    GF_FREE (crawl_event->start_time_str);
+    crawl_event->start_time_str = NULL;
+    crawl_event->end_time_str = NULL;
+    crawl_event->crawl_type = NULL;
+    crawl_event->crawl_inprogress = _gf_false;
+    return;
+}
+
+static void
+afr_copy_crawl_event_struct (shd_crawl_event_t *src, shd_crawl_event_t *dst)
+{
+        dst->healed_count = src->healed_count;
+        dst->split_brain_count = src->split_brain_count;
+        dst->heal_failed_count = src->heal_failed_count;
+        dst->start_time_str = gf_strdup (src->start_time_str);
+        dst->end_time_str = "Crawl is already in progress";
+        dst->crawl_type = src->crawl_type;
+        dst->crawl_inprogress = _gf_true;
+        return;
+}
+
+static int
+afr_fill_crawl_statistics_of_running_crawl(xlator_t *this, dict_t *dict)
+{
+        shd_crawl_event_t       *evnt = NULL;
+        int                     ret = 0;
+        afr_private_t           *priv = NULL;
+        afr_self_heald_t        *shd = NULL;
+        int                     i = 0;
+        priv = this->private;
+        shd = &priv->shd;
+
+        evnt = GF_CALLOC (1, sizeof (shd_crawl_event_t),
+                          gf_afr_mt_shd_crawl_event_t);
+        if (!evnt) {
+                ret = -1;
+                goto out;
+        }
+        LOCK (&priv->lock);
+        {
+                for (i = 0; i < priv->child_count; i++) {
+                        if (shd->pos[i] != AFR_POS_LOCAL)
+                                continue;
+
+                        reset_crawl_event (evnt);
+
+                        if (!shd->crawl_events[i]) {
+                                continue;
+                        }
+
+                        afr_copy_crawl_event_struct (shd->crawl_events[i],
+                                                     evnt);
+                        _add_crawl_stats_to_dict (this, dict, i, evnt, NULL);
+
+                }
+        }
+        UNLOCK (&priv->lock);
+        reset_crawl_event (evnt);
+        GF_FREE (evnt);
+
+out:
+        return ret;
+}
+
+static int
+_add_local_subvols_crawl_statistics_to_dict (xlator_t *this, dict_t *dict)
+{
+        int ret = 0;
+        afr_fill_completed_crawl_statistics_to_dict (this, dict);
+        ret = afr_fill_crawl_statistics_of_running_crawl (this, dict);
+        return ret;
+}
 int
-_add_all_subvols_eh_to_dict (xlator_t *this, eh_t *eh, dict_t *dict)
+_add_local_subvols_eh_to_dict (xlator_t *this, eh_t *eh, dict_t *dict)
 {
         afr_private_t           *priv = NULL;
         afr_self_heald_t        *shd = NULL;
@@ -502,7 +920,7 @@ _add_all_subvols_eh_to_dict (xlator_t *this, eh_t *eh, dict_t *dict)
         for (i = 0; i < priv->child_count; i++) {
                 if (shd->pos[i] != AFR_POS_LOCAL)
                         continue;
-                _add_eh_to_dict (this, eh, dict, shd->sh_times[i], i);
+                _add_eh_to_dict (this, eh, dict, i);
         }
         return 0;
 }
@@ -536,19 +954,29 @@ afr_xl_op (xlator_t *this, dict_t *input, dict_t *output)
                 ret = _do_self_heal_on_local_subvols (this, FULL, output);
                 break;
         case GF_AFR_OP_INDEX_SUMMARY:
-                ret = _get_index_summary_on_local_subvols (this, output);
+                (void)_get_index_summary_on_local_subvols (this, output);
+                ret = 0;
                 break;
         case GF_AFR_OP_HEALED_FILES:
-                ret = _add_all_subvols_eh_to_dict (this, shd->healed, output);
+                ret = _add_local_subvols_eh_to_dict (this, shd->healed, output);
                 break;
         case GF_AFR_OP_HEAL_FAILED_FILES:
-                ret = _add_all_subvols_eh_to_dict (this, shd->heal_failed,
+                ret = _add_local_subvols_eh_to_dict (this, shd->heal_failed,
                                                    output);
                 break;
         case GF_AFR_OP_SPLIT_BRAIN_FILES:
-                ret = _add_all_subvols_eh_to_dict (this, shd->split_brain,
+                ret = _add_local_subvols_eh_to_dict (this, shd->split_brain,
                                                    output);
                 break;
+        case GF_AFR_OP_STATISTICS:
+                ret = _add_local_subvols_crawl_statistics_to_dict (this, output);
+                break;
+        case GF_AFR_OP_STATISTICS_HEAL_COUNT:
+        case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                ret = _do_crawl_op_on_local_subvols (this, INDEX_TO_BE_HEALED,
+                                                     STATISTICS_TO_BE_HEALED,
+                                                     output);
+                break;
         default:
                 gf_log (this->name, GF_LOG_ERROR, "Unknown set op %d", op);
                 break;
@@ -563,33 +991,48 @@ afr_poll_self_heal (void *data)
 {
         afr_private_t    *priv = NULL;
         afr_self_heald_t *shd = NULL;
-        struct timeval   timeout = {0};
+        struct timespec  timeout = {0};
         xlator_t         *this = NULL;
         long             child = (long)data;
         gf_timer_t       *old_timer = NULL;
         gf_timer_t       *new_timer = NULL;
+        shd_pos_t        pos_data = {0};
+        int              ret = 0;
 
         this = THIS;
         priv = this->private;
         shd = &priv->shd;
 
-        _do_self_heal_on_subvol (this, child, INDEX);
-        timeout.tv_sec = AFR_POLL_TIMEOUT;
-        timeout.tv_usec = 0;
+        if (shd->pos[child] == AFR_POS_UNKNOWN) {
+                pos_data.this = this;
+                pos_data.child = child;
+                ret = synctask_new (this->ctx->env,
+                                    afr_syncop_find_child_position,
+                                    NULL, NULL, &pos_data);
+                if (!ret)
+                        shd->pos[child] = pos_data.pos;
+        }
+        if (shd->enabled && (shd->pos[child] == AFR_POS_LOCAL))
+                _do_self_heal_on_subvol (this, child, INDEX);
+        timeout.tv_sec = shd->timeout;
+        timeout.tv_nsec = 0;
         //notify and previous timer should be synchronized.
         LOCK (&priv->lock);
         {
                 old_timer = shd->timer[child];
+                if (shd->pos[child] == AFR_POS_REMOTE)
+                        goto unlock;
                 shd->timer[child] = gf_timer_call_after (this->ctx, timeout,
                                                          afr_poll_self_heal,
                                                          data);
                 new_timer = shd->timer[child];
         }
+unlock:
         UNLOCK (&priv->lock);
 
         if (old_timer)
                 gf_timer_call_cancel (this->ctx, old_timer);
-        if (!new_timer) {
+        if (!new_timer && (shd->pos[child] != AFR_POS_REMOTE)) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "Could not create self-heal polling timer for %s",
                         priv->children[child]->name);
@@ -598,7 +1041,7 @@ afr_poll_self_heal (void *data)
 }
 
 static int
-afr_local_child_poll_self_heal  (int ret, call_frame_t *sync_frame, void *data)
+afr_handle_child_up  (int ret, call_frame_t *sync_frame, void *data)
 {
         afr_self_heald_t *shd = NULL;
         shd_pos_t        *pos_data = data;
@@ -610,8 +1053,9 @@ afr_local_child_poll_self_heal  (int ret, call_frame_t *sync_frame, void *data)
         priv = pos_data->this->private;
         shd = &priv->shd;
         shd->pos[pos_data->child] = pos_data->pos;
-        if (pos_data->pos == AFR_POS_LOCAL)
+        if (pos_data->pos != AFR_POS_REMOTE)
                 afr_poll_self_heal ((void*)(long)pos_data->child);
+        _do_self_heal_on_local_subvols (THIS, INDEX, NULL);
 out:
         GF_FREE (data);
         return 0;
@@ -635,7 +1079,7 @@ afr_proactive_self_heal (void *data)
         pos_data->this = this;
         pos_data->child = child;
         ret = synctask_new (this->ctx->env, afr_syncop_find_child_position,
-                            afr_local_child_poll_self_heal, NULL, pos_data);
+                            afr_handle_child_up, NULL, pos_data);
         if (ret)
                 goto out;
 out:
@@ -698,7 +1142,6 @@ out:
         return ret;
 }
 
-
 int
 afr_crawl_build_start_loc (xlator_t *this, afr_crawl_data_t *crawl_data,
                            loc_t *dirloc)
@@ -706,6 +1149,7 @@ afr_crawl_build_start_loc (xlator_t *this, afr_crawl_data_t *crawl_data,
         afr_private_t *priv = NULL;
         dict_t        *xattr = NULL;
         void          *index_gfid = NULL;
+        void          *base_indices_holder_vgfid = NULL;
         loc_t         rootloc = {0};
         struct iatt   iattr = {0};
         struct iatt   parent = {0};
@@ -715,7 +1159,7 @@ afr_crawl_build_start_loc (xlator_t *this, afr_crawl_data_t *crawl_data,
         priv = this->private;
         if (crawl_data->crawl == FULL) {
                 afr_build_root_loc (this, dirloc);
-        } else {
+        } else if (crawl_data->crawl == INDEX) {
                 afr_build_root_loc (this, &rootloc);
                 ret = syncop_getxattr (readdir_xl, &rootloc, &xattr,
                                        GF_XATTROP_INDEX_GFID);
@@ -739,11 +1183,57 @@ afr_crawl_build_start_loc (xlator_t *this, afr_crawl_data_t *crawl_data,
                 ret = syncop_lookup (readdir_xl, dirloc, NULL,
                                      &iattr, NULL, &parent);
                 if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "lookup failed on "
-                                "index dir on %s", readdir_xl->name);
+                        if (errno != ENOENT) {
+                                gf_log (this->name, GF_LOG_ERROR, "lookup "
+                                        "failed on index dir on %s - (%s)",
+                                        readdir_xl->name, strerror (errno));
+                        }
+                        goto out;
+                }
+                ret = _link_inode_update_loc (this, dirloc, &iattr);
+                if (ret)
+                        goto out;
+        } else if (crawl_data->crawl == INDEX_TO_BE_HEALED) {
+                afr_build_root_loc (this, &rootloc);
+                ret = syncop_getxattr (readdir_xl, &rootloc, &xattr,
+                                       GF_BASE_INDICES_HOLDER_GFID);
+                if (ret < 0)
+                        goto out;
+                ret = dict_get_ptr (xattr, GF_BASE_INDICES_HOLDER_GFID,
+                                    &base_indices_holder_vgfid);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "index gfid empty "
+                                "on %s", readdir_xl->name);
+                        ret = -1;
+                        goto out;
+                }
+                if (!base_indices_holder_vgfid) {
+                        gf_log (this->name, GF_LOG_ERROR, "Base indices holder"
+                                "virtual gfid is null on %s", readdir_xl->name);
+                        ret = -1;
+                        goto out;
+                }
+                uuid_copy (dirloc->gfid,  base_indices_holder_vgfid);
+                dirloc->path = "";
+                dirloc->inode = inode_new (priv->root_inode->table);
+                ret = syncop_lookup (readdir_xl, dirloc, NULL, &iattr, NULL,
+                                     &parent);
+                if (ret < 0) {
+                        if (errno != ENOENT) {
+                                gf_log (this->name, GF_LOG_ERROR, "lookup "
+                                        "failed for base_indices_holder dir"
+                                        " on %s - (%s)", readdir_xl->name,
+                                        strerror (errno));
+
+                        } else {
+                                gf_log (this->name, GF_LOG_ERROR, "base_indices"
+                                        "_holder is not yet created.");
+                        }
                         goto out;
                 }
-                inode_link (dirloc->inode, NULL, NULL, &iattr);
+                ret = _link_inode_update_loc (this, dirloc, &iattr);
+                if (ret)
+                        goto out;
         }
         ret = 0;
 out:
@@ -802,17 +1292,30 @@ int
 afr_crawl_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent,
                            gf_dirent_t *entry, afr_crawl_data_t *crawl_data)
 {
-        int           ret = 0;
+        int           ret = -1;
         afr_private_t *priv = NULL;
 
         priv = this->private;
         if (crawl_data->crawl == FULL) {
                 ret = afr_build_child_loc (this, child, parent, entry->d_name);
+        } else if (crawl_data->crawl == INDEX_TO_BE_HEALED) {
+                ret = _build_index_loc (this, child, entry->d_name, parent);
+                if (ret)
+                        goto out;
+                child->inode = inode_new (priv->root_inode->table);
+                if (!child->inode) {
+                        ret = -1;
+                        goto out;
+                }
+                child->path = NULL;
         } else {
-                child->path = "";
                 child->inode = inode_new (priv->root_inode->table);
+                if (!child->inode)
+                        goto out;
                 uuid_parse (entry->d_name, child->gfid);
+                ret = _loc_assign_gfid_path (child);
         }
+out:
         return ret;
 }
 
@@ -826,7 +1329,6 @@ _process_entries (xlator_t *this, loc_t *parentloc, gf_dirent_t *entries,
         loc_t            entry_loc = {0};
         fd_t             *fd = NULL;
         struct iatt      iattr = {0};
-        inode_t          *link_inode = NULL;
 
         list_for_each_entry_safe (entry, tmp, &entries->list, list) {
                 if (!_crawl_proceed (this, crawl_data->child,
@@ -846,8 +1348,6 @@ _process_entries (xlator_t *this, loc_t *parentloc, gf_dirent_t *entries,
                         continue;
                 }
 
-                if (crawl_data->crawl == INDEX)
-                        entry_loc.path = NULL;//HACK
                 loc_wipe (&entry_loc);
                 ret = afr_crawl_build_child_loc (this, &entry_loc, parentloc,
                                                  entry, crawl_data);
@@ -857,22 +1357,18 @@ _process_entries (xlator_t *this, loc_t *parentloc, gf_dirent_t *entries,
                 ret = crawl_data->process_entry (this, crawl_data, entry,
                                                  &entry_loc, parentloc, &iattr);
 
-                if (crawl_data->crawl == INDEX)
+                if (crawl_data->crawl == INDEX_TO_BE_HEALED && ret) {
+                       goto out;
+                } else if (ret) {
                         continue;
+                }
 
-                if (ret || !IA_ISDIR (iattr.ia_type))
+                if ((crawl_data->crawl == INDEX) ||
+                    (crawl_data->crawl == INDEX_TO_BE_HEALED))
                         continue;
 
-                link_inode = inode_link (entry_loc.inode, NULL, NULL, &iattr);
-                if (link_inode == NULL) {
-                        char uuidbuf[64];
-                        gf_log (this->name, GF_LOG_ERROR, "inode link failed "
-                                "on the inode (%s)",
-                                uuid_utoa_r (entry_loc.gfid, uuidbuf));
-                        ret = -1;
-                        goto out;
-                }
-
+                if (!IA_ISDIR (iattr.ia_type))
+                        continue;
                 fd = NULL;
                 ret = afr_crawl_opendir (this, crawl_data, &fd, &entry_loc);
                 if (ret)
@@ -883,10 +1379,11 @@ _process_entries (xlator_t *this, loc_t *parentloc, gf_dirent_t *entries,
         }
         ret = 0;
 out:
-        if (crawl_data->crawl == INDEX)
-                entry_loc.path = NULL;
-        if (entry_loc.path)
-                loc_wipe (&entry_loc);
+        if ((crawl_data->crawl == INDEX_TO_BE_HEALED)  && ret) {
+                gf_log (this->name, GF_LOG_ERROR,"Failed to get the hardlink "
+                        "count");
+        }
+        loc_wipe (&entry_loc);
         return ret;
 }
 
@@ -905,10 +1402,10 @@ _crawl_directory (fd_t *fd, loc_t *loc, afr_crawl_data_t *crawl_data)
 
         GF_ASSERT (loc->inode);
 
-        if (loc->path)
+        if (crawl_data->crawl == FULL)
                 gf_log (this->name, GF_LOG_DEBUG, "crawling %s", loc->path);
         else
-                gf_log (this->name, GF_LOG_DEBUG, "crawling %s",
+                gf_log (this->name, GF_LOG_DEBUG, "crawling INDEX %s",
                         uuid_utoa (loc->gfid));
 
         while (1) {
@@ -933,6 +1430,9 @@ _crawl_directory (fd_t *fd, loc_t *loc, afr_crawl_data_t *crawl_data)
 
                 ret = _process_entries (this, loc, &entries, &offset,
                                         crawl_data);
+                if ((ret < 0) && (crawl_data->crawl == INDEX_TO_BE_HEALED)) {
+                        goto out;
+                }
                 gf_dirent_free (&entries);
                 free_entries = _gf_false;
         }
@@ -961,40 +1461,38 @@ int
 afr_find_child_position (xlator_t *this, int child, afr_child_pos_t *pos)
 {
         afr_private_t    *priv = NULL;
+        afr_self_heald_t *shd  = NULL;
         dict_t           *xattr_rsp = NULL;
         loc_t            loc = {0};
         int              ret = 0;
-        gf_boolean_t     local = _gf_false;
-        char             *pathinfo = NULL;
+        char             *node_uuid = NULL;
 
         priv = this->private;
+        shd  = &priv->shd;
 
         afr_build_root_loc (this, &loc);
 
         ret = syncop_getxattr (priv->children[child], &loc, &xattr_rsp,
-                               GF_XATTR_PATHINFO_KEY);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "getxattr failed on %s",
-                        priv->children[child]->name);
+                               GF_XATTR_NODE_UUID_KEY);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "getxattr failed on %s - "
+                        "(%s)", priv->children[child]->name, strerror (errno));
                 goto out;
         }
 
-        ret = dict_get_str (xattr_rsp, GF_XATTR_PATHINFO_KEY, &pathinfo);
+        ret = dict_get_str (xattr_rsp, GF_XATTR_NODE_UUID_KEY, &node_uuid);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Pathinfo key not found on "
-                        "child %d", child);
+                gf_log (this->name, GF_LOG_ERROR, "node-uuid key not found on "
+                        "child %s", priv->children[child]->name);
                 goto out;
         }
 
-        ret = afr_local_pathinfo (pathinfo, &local);
-        if (ret)
-                goto out;
-        if (local)
+        if (!strcmp (node_uuid, shd->node_uuid))
                 *pos = AFR_POS_LOCAL;
         else
                 *pos = AFR_POS_REMOTE;
 
-        gf_log (this->name, GF_LOG_INFO, "child %s is %s",
+        gf_log (this->name, GF_LOG_DEBUG, "child %s is %s",
                 priv->children[child]->name, position_str_get (*pos));
 out:
         if (ret)
@@ -1040,27 +1538,122 @@ afr_dir_crawl (void *data)
                 goto out;
 
         ret = afr_crawl_opendir (this, crawl_data, &fd, &dirloc);
-        if (ret)
+        if (ret) {
+                if (crawl_data->crawl == INDEX_TO_BE_HEALED) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to open base_"
+                                "indices_holder");
+                }
                 goto out;
+        }
 
         ret = _crawl_directory (fd, &dirloc, crawl_data);
         if (ret)
                 gf_log (this->name, GF_LOG_ERROR, "Crawl failed on %s",
                         readdir_xl->name);
         else
-                gf_log (this->name, GF_LOG_INFO, "Crawl completed "
+                gf_log (this->name, GF_LOG_DEBUG, "Crawl completed "
                         "on %s", readdir_xl->name);
         if (crawl_data->crawl == INDEX)
                 dirloc.path = NULL;
 out:
         if (fd)
                 fd_unref (fd);
-        if (crawl_data->crawl == INDEX)
+        if ((crawl_data->crawl == INDEX) ||
+            (crawl_data->crawl == INDEX_TO_BE_HEALED ))
                 dirloc.path = NULL;
         loc_wipe (&dirloc);
         return ret;
 }
 
+char *
+get_crawl_type_in_string (afr_crawl_type_t crawl)
+{
+        char    *index = "INDEX";
+        char    *full  = "FULL";
+        char    *crawl_type = NULL;
+
+        if (crawl == INDEX){
+                crawl_type = index;
+        } else if (crawl == FULL) {
+                crawl_type = full;
+        }
+
+        return  crawl_type;
+}
+
+static int
+afr_allocate_crawl_event (xlator_t *this, int child, afr_crawl_type_t crawl)
+{
+        afr_private_t           *priv = NULL;
+        afr_self_heald_t        *shd = NULL;
+        int                     ret = 0;
+        shd_crawl_event_t       *crawl_event = NULL;
+        time_t                  get_time = 0;
+
+        priv = this->private;
+        shd = &priv->shd;
+
+        crawl_event = GF_CALLOC (sizeof (shd_crawl_event_t), 1,
+                                 gf_afr_mt_shd_crawl_event_t);
+        if (!crawl_event) {
+                ret = -1;
+                goto out;
+        }
+
+        get_time =  time(NULL);
+        if (get_time == ((time_t)-1)) {
+                 ret = -1;
+                goto out;
+        }
+
+        crawl_event->start_time_str = gf_strdup (ctime(&get_time));
+
+        crawl_event->crawl_type = get_crawl_type_in_string (crawl);
+        if (!crawl_event->crawl_type) {
+                ret = -1;
+                goto out;
+        }
+        LOCK (&priv->lock);
+        {
+                shd->crawl_events[child] = crawl_event;
+        }
+        UNLOCK (&priv->lock);
+        ret = 0;
+out:
+        return ret;
+
+}
+
+static int
+afr_put_crawl_event_in_eh (xlator_t *this, int child)
+{
+        afr_private_t           *priv = NULL;
+        afr_self_heald_t        *shd = NULL;
+        int                     ret = 0;
+        time_t                  get_time = 0;
+        shd_crawl_event_t       **crawl_event = NULL;
+
+        priv = this->private;
+        shd = &priv->shd;
+
+        get_time = time(NULL);
+        if (get_time == ((time_t)-1)) {
+                ret = -1;
+                goto out;
+        }
+        crawl_event = (shd_crawl_event_t**)shd->crawl_events;
+        LOCK (&priv->lock);
+        {
+                crawl_event[child]->end_time_str = gf_strdup (ctime(&get_time));
+                ret = eh_save_history (shd->statistics[child],
+                                       crawl_event[child]);
+                crawl_event[child] = NULL;
+        }
+        UNLOCK (&priv->lock);
+out:
+        return ret;
+}
+
 static int
 afr_dir_exclusive_crawl (void *data)
 {
@@ -1096,7 +1689,15 @@ afr_dir_exclusive_crawl (void *data)
         }
 
         do {
+                ret = afr_allocate_crawl_event (this, child, crawl_data->crawl);
+                if (ret)
+                        goto out;
                 afr_dir_crawl (data);
+
+                ret = afr_put_crawl_event_in_eh (this, child);
+                if (ret < 0)
+                        goto out;
+
                 LOCK (&priv->lock);
                 {
                         if (shd->pending[child] != NONE) {
@@ -1143,7 +1744,7 @@ afr_start_crawl (xlator_t *this, int idx, afr_crawl_type_t crawl,
         crawl_data->crawl = crawl;
         crawl_data->op_data = op_data;
         crawl_data->crawl_flags = crawl_flags;
-        gf_log (this->name, GF_LOG_INFO, "starting crawl %d for %s",
+        gf_log (this->name, GF_LOG_DEBUG, "starting crawl %d for %s",
                 crawl_data->crawl, priv->children[idx]->name);
 
         if (exclusive)
@@ -1153,8 +1754,8 @@ afr_start_crawl (xlator_t *this, int idx, afr_crawl_type_t crawl,
         ret = synctask_new (this->ctx->env, crawler,
                             crawl_done, frame, crawl_data);
         if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Could not create the "
-                        "task for %d ret %d", idx, ret);
+                gf_log (this->name, GF_LOG_ERROR, "afr crawl failed for child"
+                        " %d with ret %d", idx, ret);
 out:
         return;
 }
@@ -1184,4 +1785,3 @@ afr_set_root_gfid (dict_t *dict)
 
         return ret;
 }
-
diff --git a/xlators/cluster/afr/src/afr-self-heald.h b/xlators/cluster/afr/src/afr-self-heald.h
index 848738621..e0c083754 100644
--- a/xlators/cluster/afr/src/afr-self-heald.h
+++ b/xlators/cluster/afr/src/afr-self-heald.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __AFR_SELF_HEALD_H__
@@ -38,6 +29,19 @@ typedef struct afr_crawl_data_ {
                               struct iatt *iattr);
 } afr_crawl_data_t;
 
+typedef struct crawl_event_stats_ {
+        uint64_t healed_count;
+        uint64_t split_brain_count;
+        uint64_t heal_failed_count;
+        char     *start_time_str;
+        char     *end_time_str;
+        char     *crawl_type;
+        gf_boolean_t crawl_inprogress;
+} shd_crawl_event_t;
+
+void _destroy_crawl_event_data (void *data);
+void _destroy_shd_event_data (void *data);
+
 typedef int (*process_entry_cbk_t) (xlator_t *this, afr_crawl_data_t *crawl_data,
                               gf_dirent_t *entry, loc_t *child, loc_t *parent,
                               struct iatt *iattr);
@@ -51,4 +55,11 @@ afr_proactive_self_heal (void *data);
 
 int
 afr_xl_op (xlator_t *this, dict_t *input, dict_t *output);
+
+/*
+ * In addition to its self-heal use, this is used to find a local default
+ * read_child.
+ */
+int
+afr_local_pathinfo (char *pathinfo, gf_boolean_t *local);
 #endif /* __AFR_SELF_HEALD_H__ */
diff --git a/xlators/cluster/afr/src/afr-transaction.c b/xlators/cluster/afr/src/afr-transaction.c
index 584ad7aed..20306e469 100644
--- a/xlators/cluster/afr/src/afr-transaction.c
+++ b/xlators/cluster/afr/src/afr-transaction.c
@@ -1,25 +1,17 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "dict.h"
 #include "byte-order.h"
 #include "common-utils.h"
+#include "timer.h"
 
 #include "afr.h"
 #include "afr-transaction.h"
@@ -32,7 +24,6 @@
                                       of RENAME */
 #define LOCKED_LOWER    0x2        /* for lower_path of RENAME */
 
-
 afr_fd_ctx_t *
 __afr_fd_ctx_get (fd_t *fd, xlator_t *this)
 {
@@ -82,27 +73,26 @@ afr_fd_ctx_get (fd_t *fd, xlator_t *this)
 
 
 static void
-afr_pid_save (call_frame_t *frame)
+afr_save_lk_owner (call_frame_t *frame)
 {
         afr_local_t * local = NULL;
 
         local = frame->local;
 
-        local->saved_pid = frame->root->pid;
+        local->saved_lk_owner = frame->root->lk_owner;
 }
 
 
 static void
-afr_pid_restore (call_frame_t *frame)
+afr_restore_lk_owner (call_frame_t *frame)
 {
         afr_local_t * local = NULL;
 
         local = frame->local;
 
-        frame->root->pid = local->saved_pid;
+        frame->root->lk_owner = local->saved_lk_owner;
 }
 
-
 static void
 __mark_all_pending (int32_t *pending[], int child_count,
                     afr_transaction_type type)
@@ -155,51 +145,23 @@ out:
         return;
 }
 
-
-static void
-__mark_pre_op_undone_on_fd (call_frame_t *frame, xlator_t *this, int child_index)
-{
-        afr_local_t   *local = NULL;
-        afr_fd_ctx_t  *fd_ctx = NULL;
-
-        local = frame->local;
-
-        if (!local->fd)
-                return;
-
-        fd_ctx = afr_fd_ctx_get (local->fd, this);
-
-        if (!fd_ctx)
-                goto out;
-
-        LOCK (&local->fd->lock);
-        {
-                if (local->transaction.type == AFR_DATA_TRANSACTION)
-                        fd_ctx->pre_op_done[child_index]--;
-        }
-        UNLOCK (&local->fd->lock);
-out:
-        return;
-}
-
-
 static void
-__mark_down_children (int32_t *pending[], int child_count,
-                      unsigned char *child_up, afr_transaction_type type)
+__mark_non_participant_children (int32_t *pending[], int child_count,
+                                 unsigned char *participants,
+                                 afr_transaction_type type)
 {
         int i = 0;
         int j = 0;
 
+        j = afr_index_for_transaction_type (type);
         for (i = 0; i < child_count; i++) {
-                j = afr_index_for_transaction_type (type);
-
-                if (!child_up[i])
+                if (!participants[i])
                         pending[i][j] = 0;
         }
 }
 
 
-static void
+void
 __mark_all_success (int32_t *pending[], int child_count,
                     afr_transaction_type type)
 {
@@ -212,6 +174,54 @@ __mark_all_success (int32_t *pending[], int child_count,
         }
 }
 
+void
+_set_all_child_errno (int *child_errno, unsigned int child_count)
+{
+        int     i = 0;
+
+        for (i = 0; i < child_count; i++)
+                if (child_errno[i] == 0)
+                        child_errno[i] = ENOTCONN;
+}
+
+void
+afr_transaction_perform_fop (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        afr_private_t   *priv = NULL;
+        fd_t            *fd   = NULL;
+
+        local = frame->local;
+        priv  = this->private;
+        fd    = local->fd;
+
+        __mark_all_success (local->pending, priv->child_count,
+                            local->transaction.type);
+
+        _set_all_child_errno (local->child_errno, priv->child_count);
+
+        /*  Perform fops with the lk-owner from top xlator.
+         *  Eg: lk-owner of posix-lk and flush should be same,
+         *  flush cant clear the  posix-lks without that lk-owner.
+         */
+        afr_save_lk_owner (frame);
+        frame->root->lk_owner =
+                local->transaction.main_frame->root->lk_owner;
+
+
+        /* The wake up needs to happen independent of
+           what type of fop arrives here. If it was
+           a write, then it has already inherited the
+           lock and changelog. If it was not a write,
+           then the presumption of the optimization (of
+           optimizing for successive write operations)
+           fails.
+        */
+        if (fd)
+                afr_delayed_changelog_wake_up (this, fd);
+        local->transaction.fop (frame, this);
+}
+
 
 static int
 __changelog_enabled (afr_private_t *priv, afr_transaction_type type)
@@ -276,63 +286,41 @@ __fop_changelog_needed (call_frame_t *frame, xlator_t *this)
 }
 
 int
-afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending)
+afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending,
+                      int child, afr_xattrop_type_t op)
 {
         int i = 0;
         int ret = 0;
 
+        if (op == LOCAL_FIRST) {
+                ret = dict_set_static_bin (xattr, priv->pending_key[child],
+                                           pending[child],
+                                   AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
+                if (ret)
+                        goto out;
+        }
         for (i = 0; i < priv->child_count; i++) {
+                if (i == child)
+                        continue;
                 ret = dict_set_static_bin (xattr, priv->pending_key[i],
                                            pending[i],
-                                AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
+                                   AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
                 /* 3 = data+metadata+entry */
 
                 if (ret < 0)
                         goto out;
         }
-
-out:
-        return ret;
-}
-
-
-static int
-afr_set_piggyback_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending,
-                        afr_transaction_type type)
-{
-        int i = 0;
-        int ret = 0;
-        int *arr = NULL;
-        int index = 0;
-        size_t pending_xattr_size = 3 * sizeof (int32_t);
-        /* 3 = data+metadata+entry */
-
-        index = afr_index_for_transaction_type (type);
-
-        for (i = 0; i < priv->child_count; i++) {
-                arr = GF_CALLOC (1, pending_xattr_size,
-                                 gf_afr_mt_char);
-                if (!arr) {
-                        ret = -1;
-                        goto out;
-                }
-
-                memcpy (arr, pending[i], pending_xattr_size);
-
-                arr[index] = hton32 (ntoh32(arr[index]) + 1);
-
-                ret = dict_set_bin (xattr, priv->pending_key[i],
-                                    arr, pending_xattr_size);
-
-                if (ret < 0)
+        if (op == LOCAL_LAST) {
+                ret = dict_set_static_bin (xattr, priv->pending_key[child],
+                                           pending[child],
+                                   AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
+                if (ret)
                         goto out;
         }
-
 out:
         return ret;
 }
 
-
 int
 afr_lock_server_count (afr_private_t *priv, afr_transaction_type type)
 {
@@ -360,7 +348,8 @@ afr_lock_server_count (afr_private_t *priv, afr_transaction_type type)
 
 int32_t
 afr_changelog_post_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, dict_t *xattr)
+                           int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                           dict_t *xdata)
 {
         afr_internal_lock_t *int_lock = NULL;
         afr_private_t       *priv     = NULL;
@@ -378,6 +367,11 @@ afr_changelog_post_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         UNLOCK (&frame->lock);
 
         if (call_count == 0) {
+                if (local->transaction.resume_stub) {
+			call_resume (local->transaction.resume_stub);
+                        local->transaction.resume_stub = NULL;
+                }
+
                 if (afr_lock_server_count (priv, local->transaction.type) == 0) {
                         local->transaction.done (frame, this);
                 } else {
@@ -444,24 +438,41 @@ afr_transaction_rm_stale_children (call_frame_t *frame, xlator_t *this,
 
         afr_inode_rm_stale_children (this, inode, stale_children);
 out:
-        if (stale_children)
-                GF_FREE (stale_children);
+        GF_FREE (stale_children);
         return;
 }
 
+afr_inodelk_t*
+afr_get_inodelk (afr_internal_lock_t *int_lock, char *dom)
+{
+        afr_inodelk_t *inodelk = NULL;
+        int           i = 0;
+
+        for (i = 0; int_lock->inodelk[i].domain; i++) {
+                inodelk = &int_lock->inodelk[i];
+                if (strcmp (dom, inodelk->domain) == 0)
+                        return inodelk;
+        }
+        return NULL;
+}
+
 unsigned char*
 afr_locked_nodes_get (afr_transaction_type type, afr_internal_lock_t *int_lock)
 {
         unsigned char *locked_nodes = NULL;
+        afr_inodelk_t *inodelk = NULL;
         switch (type) {
         case AFR_DATA_TRANSACTION:
         case AFR_METADATA_TRANSACTION:
-                locked_nodes = int_lock->inode_locked_nodes;
+                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+                locked_nodes = inodelk->locked_nodes;
         break;
 
         case AFR_ENTRY_TRANSACTION:
         case AFR_ENTRY_RENAME_TRANSACTION:
-                locked_nodes = int_lock->entry_locked_nodes;
+                /*Because same set of subvols participate in all lockee
+                 * entities*/
+                locked_nodes = int_lock->lockee[0].locked_nodes;
         break;
         }
         return locked_nodes;
@@ -499,12 +510,141 @@ afr_changelog_post_op_call_count (afr_transaction_type type,
         return call_count;
 }
 
+void
+afr_compute_txn_changelog (afr_local_t *local, afr_private_t *priv)
+{
+        int             i = 0;
+        int             index = 0;
+        int32_t         postop = 0;
+        int32_t         preop = 1;
+        int32_t         **txn_changelog = NULL;
+
+        txn_changelog = local->transaction.txn_changelog;
+        index = afr_index_for_transaction_type (local->transaction.type);
+        for (i = 0; i < priv->child_count; i++) {
+                postop = ntoh32 (local->pending[i][index]);
+                txn_changelog[i][index] = hton32 (postop + preop);
+        }
+}
+
+afr_xattrop_type_t
+afr_get_postop_xattrop_type (int32_t **pending, int optimized, int child,
+                             afr_transaction_type type)
+{
+        int                     index = 0;
+        afr_xattrop_type_t      op = LOCAL_LAST;
+
+        index = afr_index_for_transaction_type (type);
+        if (optimized && !pending[child][index])
+                op = LOCAL_FIRST;
+        return op;
+}
+
+void
+afr_set_postop_dict (afr_local_t *local, xlator_t *this, dict_t *xattr,
+                     int optimized, int child)
+{
+        int32_t                 **txn_changelog = NULL;
+        int32_t                 **changelog = NULL;
+        afr_private_t           *priv = NULL;
+        int                     ret = 0;
+        afr_xattrop_type_t      op = LOCAL_LAST;
+
+        priv = this->private;
+        txn_changelog = local->transaction.txn_changelog;
+        op = afr_get_postop_xattrop_type (local->pending, optimized, child,
+                                          local->transaction.type);
+        if (optimized)
+                changelog = txn_changelog;
+        else
+                changelog = local->pending;
+        ret = afr_set_pending_dict (priv, xattr, changelog, child, op);
+        if (ret < 0)
+                gf_log (this->name, GF_LOG_INFO,
+                        "failed to set pending entry");
+}
+
+
+gf_boolean_t
+afr_txn_nothing_failed (call_frame_t *frame, xlator_t *this)
+{
+        afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+        int index = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        index = afr_index_for_transaction_type (local->transaction.type);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->pending[i][index] == 0)
+                        return _gf_false;
+        }
+
+        return _gf_true;
+}
+
+static void
+afr_dir_fop_handle_all_fop_failures (call_frame_t *frame)
+{
+        xlator_t        *this = NULL;
+        afr_local_t     *local = NULL;
+        afr_private_t   *priv = NULL;
+
+        this = frame->this;
+        local = frame->local;
+        priv = this->private;
+
+        if ((local->transaction.type != AFR_ENTRY_TRANSACTION) &&
+            (local->transaction.type != AFR_ENTRY_RENAME_TRANSACTION))
+                return;
+
+        if (local->op_ret >= 0)
+                goto out;
+
+        __mark_all_success (local->pending, priv->child_count,
+                            local->transaction.type);
+out:
+        return;
+}
+
+static void
+afr_data_handle_quota_errors (call_frame_t *frame, xlator_t *this)
+{
+        int     i = 0;
+        afr_private_t *priv = NULL;
+        afr_local_t   *local = NULL;
+        gf_boolean_t  all_quota_failures = _gf_false;
+
+        local = frame->local;
+        priv  = this->private;
+        if (local->transaction.type != AFR_DATA_TRANSACTION)
+                return;
+        /*
+         * Idea is to not leave the file in FOOL-FOOL scenario in case on
+         * all the bricks data transaction failed with EDQUOT to avoid
+         * increasing un-necessary load of self-heals in the system.
+         */
+        all_quota_failures = _gf_true;
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i] &&
+                    (local->child_errno[i] != EDQUOT)) {
+                        all_quota_failures = _gf_false;
+                        break;
+                }
+        }
+        if (all_quota_failures)
+                __mark_all_success (local->pending, priv->child_count,
+                                    local->transaction.type);
+}
+
 int
-afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
+afr_changelog_post_op_now (call_frame_t *frame, xlator_t *this)
 {
         afr_private_t * priv = this->private;
         afr_internal_lock_t *int_lock = NULL;
-        int ret        = 0;
         int i          = 0;
         int call_count = 0;
 
@@ -512,14 +652,17 @@ afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
         afr_fd_ctx_t  *fdctx = NULL;
         dict_t        **xattr = NULL;
         int            piggyback = 0;
-        int            index = 0;
         int            nothing_failed = 1;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
 
-        __mark_down_children (local->pending, priv->child_count,
-                              local->child_up, local->transaction.type);
+        __mark_non_participant_children (local->pending, priv->child_count,
+                                         local->transaction.pre_op,
+                                         local->transaction.type);
+
+        afr_data_handle_quota_errors (frame, this);
+        afr_dir_fop_handle_all_fop_failures (frame);
 
         if (local->fd)
                 afr_transaction_rm_stale_children (frame, this,
@@ -547,81 +690,65 @@ afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
                 goto out;
         }
 
-        /* check if something has failed, to handle piggybacking */
-        nothing_failed = 1;
-        index = afr_index_for_transaction_type (local->transaction.type);
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->pending[i][index] == 0) {
-                        nothing_failed = 0;
-                        break;
-                }
-        }
+        nothing_failed = afr_txn_nothing_failed (frame, this);
 
-        index = afr_index_for_transaction_type (local->transaction.type);
-        if (local->optimistic_change_log &&
-            local->transaction.type != AFR_DATA_TRANSACTION) {
-                /* if nothing_failed, then local->pending[..] == {0 .. 0} */
-                for (i = 0; i < priv->child_count; i++)
-                        local->pending[i][index]++;
-        }
+        afr_compute_txn_changelog (local , priv);
 
         for (i = 0; i < priv->child_count; i++) {
                 if (!local->transaction.pre_op[i])
                         continue;
-                ret = afr_set_pending_dict (priv, xattr[i], local->pending);
-
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "failed to set pending entry");
-
 
+                if (local->transaction.type != AFR_DATA_TRANSACTION)
+                        afr_set_postop_dict (local, this, xattr[i],
+                                             local->optimistic_change_log, i);
                 switch (local->transaction.type) {
                 case AFR_DATA_TRANSACTION:
                 {
                         if (!fdctx) {
+                                afr_set_postop_dict (local, this, xattr[i],
+                                                     0, i);
                                 STACK_WIND (frame, afr_changelog_post_op_cbk,
                                             priv->children[i],
                                             priv->children[i]->fops->xattrop,
                                             &local->loc,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                            GF_XATTROP_ADD_ARRAY, xattr[i],
+                                            NULL);
                                 break;
                         }
 
-                        LOCK (&local->fd->lock);
-                        {
-                                piggyback = 0;
-                                if (fdctx->pre_op_piggyback[i]) {
-                                        fdctx->pre_op_piggyback[i]--;
-                                        piggyback = 1;
-                                }
-                        }
-                        UNLOCK (&local->fd->lock);
+                        /* local->transaction.postop_piggybacked[] was
+                           precomputed in is_piggyback_postop() when called from
+                           afr_changelog_post_op_safe()
+                        */
 
-                        if (piggyback && !nothing_failed)
-                                ret = afr_set_piggyback_dict (priv, xattr[i],
-                                                              local->pending,
-                                                              local->transaction.type);
+                        piggyback = 0;
+                        if (local->transaction.postop_piggybacked[i])
+                                piggyback = 1;
+
+                        afr_set_postop_dict (local, this, xattr[i],
+                                             piggyback, i);
 
                         if (nothing_failed && piggyback) {
                                 afr_changelog_post_op_cbk (frame, (void *)(long)i,
-                                                           this, 1, 0, xattr[i]);
+                                                           this, 1, 0, xattr[i], NULL);
                         } else {
-                                __mark_pre_op_undone_on_fd (frame, this, i);
                                 STACK_WIND_COOKIE (frame,
                                                    afr_changelog_post_op_cbk,
                                                    (void *) (long) i,
                                                    priv->children[i],
                                                    priv->children[i]->fops->fxattrop,
                                                    local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
                         }
                 }
                 break;
                 case AFR_METADATA_TRANSACTION:
                 {
-                        if (nothing_failed) {
+                        if (nothing_failed && local->optimistic_change_log) {
                                 afr_changelog_post_op_cbk (frame, (void *)(long)i,
-                                                           this, 1, 0, xattr[i]);
+                                                           this, 1, 0, xattr[i],
+                                                           NULL);
                                 break;
                         }
 
@@ -630,28 +757,32 @@ afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
                                             priv->children[i],
                                             priv->children[i]->fops->fxattrop,
                                             local->fd,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                            GF_XATTROP_ADD_ARRAY, xattr[i],
+                                            NULL);
                         else
                                 STACK_WIND (frame, afr_changelog_post_op_cbk,
                                             priv->children[i],
                                             priv->children[i]->fops->xattrop,
                                             &local->loc,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                            GF_XATTROP_ADD_ARRAY, xattr[i],
+                                            NULL);
                 }
                 break;
 
                 case AFR_ENTRY_RENAME_TRANSACTION:
                 {
-                        if (nothing_failed) {
+                        if (nothing_failed && local->optimistic_change_log) {
                                 afr_changelog_post_op_cbk (frame, (void *)(long)i,
-                                                           this, 1, 0, xattr[i]);
+                                                           this, 1, 0, xattr[i],
+                                                           NULL);
                         } else {
                                 STACK_WIND_COOKIE (frame, afr_changelog_post_op_cbk,
                                                    (void *) (long) i,
                                                    priv->children[i],
                                                    priv->children[i]->fops->xattrop,
                                                    &local->transaction.new_parent_loc,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
                         }
                         call_count--;
                 }
@@ -664,19 +795,17 @@ afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
                   value
                 */
 
-                ret = afr_set_pending_dict (priv, xattr[i], local->pending);
-
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "failed to set pending entry");
+                afr_set_postop_dict (local, this, xattr[i],
+                                     local->optimistic_change_log, i);
 
                 /* fall through */
 
                 case AFR_ENTRY_TRANSACTION:
                 {
-                        if (nothing_failed) {
+                        if (nothing_failed && local->optimistic_change_log) {
                                 afr_changelog_post_op_cbk (frame, (void *)(long)i,
-                                                           this, 1, 0, xattr[i]);
+                                                           this, 1, 0, xattr[i],
+                                                           NULL);
                                 break;
                         }
 
@@ -685,13 +814,15 @@ afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
                                             priv->children[i],
                                             priv->children[i]->fops->fxattrop,
                                             local->fd,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                            GF_XATTROP_ADD_ARRAY, xattr[i],
+                                            NULL);
                         else
                                 STACK_WIND (frame, afr_changelog_post_op_cbk,
                                             priv->children[i],
                                             priv->children[i]->fops->xattrop,
                                             &local->transaction.parent_loc,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                            GF_XATTROP_ADD_ARRAY, xattr[i],
+                                            NULL);
                 }
                 break;
                 }
@@ -711,7 +842,8 @@ out:
 
 int32_t
 afr_changelog_pre_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xattr)
+                          int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                          dict_t *xdata)
 {
         afr_local_t *   local = NULL;
         afr_private_t * priv  = this->private;
@@ -756,12 +888,7 @@ afr_changelog_pre_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     (local->op_errno == ENOTSUP)) {
                         local->transaction.resume (frame, this);
                 } else {
-                        __mark_all_success (local->pending, priv->child_count,
-                                            local->transaction.type);
-
-                        afr_pid_restore (frame);
-
-                        local->transaction.fop (frame, this);
+                        afr_transaction_perform_fop (frame, this);
                 }
         }
 
@@ -814,7 +941,8 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
         for (i = 0; i < priv->child_count; i++) {
                 if (!locked_nodes[i])
                         continue;
-                ret = afr_set_pending_dict (priv, xattr[i], local->pending);
+                ret = afr_set_pending_dict (priv, xattr[i], local->pending,
+                                            i, LOCAL_FIRST);
 
                 if (ret < 0)
                         gf_log (this->name, GF_LOG_INFO,
@@ -831,7 +959,8 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                                                    priv->children[i],
                                                    priv->children[i]->fops->xattrop,
                                                    &(local->loc),
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
                                 break;
                         }
 
@@ -848,9 +977,12 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                         }
                         UNLOCK (&local->fd->lock);
 
+                        afr_set_delayed_post_op (frame, this);
+
                         if (piggyback)
                                 afr_changelog_pre_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i]);
+                                                          this, 1, 0, xattr[i],
+                                                          NULL);
                         else
                                 STACK_WIND_COOKIE (frame,
                                                    afr_changelog_pre_op_cbk,
@@ -858,14 +990,16 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                                                    priv->children[i],
                                                    priv->children[i]->fops->fxattrop,
                                                    local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
                 }
                 break;
                 case AFR_METADATA_TRANSACTION:
                 {
                         if (local->optimistic_change_log) {
                                 afr_changelog_pre_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i]);
+                                                          this, 1, 0, xattr[i],
+                                                          NULL);
                                 break;
                         }
 
@@ -876,7 +1010,8 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                                                    priv->children[i],
                                                    priv->children[i]->fops->fxattrop,
                                                    local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
                         else
                                 STACK_WIND_COOKIE (frame,
                                                    afr_changelog_pre_op_cbk,
@@ -884,7 +1019,8 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                                                    priv->children[i],
                                                    priv->children[i]->fops->xattrop,
                                                    &(local->loc),
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
                 }
                 break;
 
@@ -892,7 +1028,8 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                 {
                         if (local->optimistic_change_log) {
                                 afr_changelog_pre_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i]);
+                                                          this, 1, 0, xattr[i],
+                                                          NULL);
                         } else {
                                 STACK_WIND_COOKIE (frame,
                                                    afr_changelog_pre_op_cbk,
@@ -900,7 +1037,8 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                                                    priv->children[i],
                                                    priv->children[i]->fops->xattrop,
                                                    &local->transaction.new_parent_loc,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
                         }
 
                         call_count--;
@@ -915,7 +1053,8 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                   value
                 */
 
-                ret = afr_set_pending_dict (priv, xattr[i], local->pending);
+                ret = afr_set_pending_dict (priv, xattr[i], local->pending,
+                                            i, LOCAL_FIRST);
 
                 if (ret < 0)
                         gf_log (this->name, GF_LOG_INFO,
@@ -927,7 +1066,8 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                 {
                         if (local->optimistic_change_log) {
                                 afr_changelog_pre_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i]);
+                                                          this, 1, 0, xattr[i],
+                                                          NULL);
                                 break;
                         }
 
@@ -938,7 +1078,8 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                                                    priv->children[i],
                                                    priv->children[i]->fops->fxattrop,
                                                    local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
                         else
                                 STACK_WIND_COOKIE (frame,
                                                    afr_changelog_pre_op_cbk,
@@ -946,7 +1087,8 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
                                                    priv->children[i],
                                                    priv->children[i]->fops->xattrop,
                                                    &local->transaction.parent_loc,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
                 }
                 break;
                 }
@@ -1001,7 +1143,6 @@ afr_post_nonblocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
                 gf_log (this->name, GF_LOG_DEBUG,
                         "Non blocking inodelks failed. Proceeding to blocking");
                 int_lock->lock_cbk = afr_post_blocking_inodelk_cbk;
-                afr_set_lk_owner (frame, this, frame->root);
                 afr_blocking_lock (frame, this);
         } else {
 
@@ -1109,12 +1250,14 @@ int
 afr_set_transaction_flock (afr_local_t *local)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
 
         int_lock = &local->internal_lock;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
 
-        int_lock->lk_flock.l_len   = local->transaction.len;
-        int_lock->lk_flock.l_start = local->transaction.start;
-        int_lock->lk_flock.l_type  = F_WRLCK;
+        inodelk->flock.l_len   = local->transaction.len;
+        inodelk->flock.l_start = local->transaction.start;
+        inodelk->flock.l_type  = F_WRLCK;
 
         return 0;
 }
@@ -1129,6 +1272,7 @@ afr_lock_rec (call_frame_t *frame, xlator_t *this)
         int_lock = &local->internal_lock;
 
         int_lock->transaction_lk_type = AFR_TRANSACTION_LK;
+        int_lock->domain = this->name;
 
         switch (local->transaction.type) {
         case AFR_DATA_TRANSACTION:
@@ -1142,8 +1286,8 @@ afr_lock_rec (call_frame_t *frame, xlator_t *this)
 
         case AFR_ENTRY_RENAME_TRANSACTION:
 
-                int_lock->lock_cbk = afr_post_blocking_rename_cbk;
-                afr_blocking_lock (frame, this);
+                int_lock->lock_cbk = afr_post_nonblocking_entrylk_cbk;
+                afr_nonblocking_entrylk (frame, this);
                 break;
 
         case AFR_ENTRY_TRANSACTION:
@@ -1165,12 +1309,6 @@ afr_lock_rec (call_frame_t *frame, xlator_t *this)
 int
 afr_lock (call_frame_t *frame, xlator_t *this)
 {
-        afr_pid_save (frame);
-
-        frame->root->pid = (long) frame->root;
-
-        afr_set_lk_owner (frame, this, frame->root);
-
         afr_set_lock_number (frame, this);
 
         return afr_lock_rec (frame, this);
@@ -1182,28 +1320,463 @@ afr_lock (call_frame_t *frame, xlator_t *this)
 int
 afr_internal_lock_finish (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
+        if (__fop_changelog_needed (frame, this)) {
+                afr_changelog_pre_op (frame, this);
+        } else {
+                afr_transaction_perform_fop (frame, this);
+        }
+
+        return 0;
+}
+
+
+void
+afr_set_delayed_post_op (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t    *local = NULL;
+        afr_private_t  *priv = NULL;
+
+        /* call this function from any of the related optimizations
+           which benefit from delaying post op are enabled, namely:
+
+           - changelog piggybacking
+           - eager locking
+        */
+
+        priv = this->private;
+        if (!priv)
+                return;
+
+        if (!priv->post_op_delay_secs)
+                return;
+
+        local = frame->local;
+        if (!local->transaction.eager_lock_on)
+                return;
+
+        if (!local)
+                return;
+
+        if (!local->fd)
+                return;
+
+        if (local->op == GF_FOP_WRITE)
+                local->delayed_post_op = _gf_true;
+}
+
+gf_boolean_t
+afr_are_multiple_fds_opened (inode_t *inode, xlator_t *this)
+{
+        afr_inode_ctx_t *ictx = NULL;
+
+        if (!inode) {
+                /* If false is returned, it may keep on taking eager-lock
+                 * which may lead to starvation, so return true to avoid that.
+                 */
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "Invalid inode");
+                return _gf_true;
+        }
+        /* Lets say mount1 has eager-lock(full-lock) and after the eager-lock
+         * is taken mount2 opened the same file, it won't be able to
+         * perform any data operations until mount1 releases eager-lock.
+         * To avoid such scenario do not enable eager-lock for this transaction
+         * if open-fd-count is > 1
+         */
+
+        ictx = afr_inode_ctx_get (inode, this);
+        if (!ictx)
+                return _gf_true;
+
+        if (ictx->open_fd_count > 1)
+                return _gf_true;
+
+        return _gf_false;
+}
+
+gf_boolean_t
+afr_any_fops_failed (afr_local_t *local, afr_private_t *priv)
+{
+        if (local->success_count != priv->child_count)
+                return _gf_true;
+        return _gf_false;
+}
+
+gf_boolean_t
+is_afr_delayed_changelog_post_op_needed (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t      *local = NULL;
+        gf_boolean_t      res = _gf_false;
+        afr_private_t    *priv  = NULL;
 
         priv  = this->private;
+
         local = frame->local;
+        if (!local)
+                goto out;
 
-        if (__fop_changelog_needed (frame, this)) {
-                afr_changelog_pre_op (frame, this);
-        } else {
-                __mark_all_success (local->pending, priv->child_count,
-                                    local->transaction.type);
+        if (!local->delayed_post_op)
+                goto out;
+
+        //Mark pending changelog ASAP
+        if (afr_any_fops_failed (local, priv))
+                goto out;
+
+        if (local->fd && afr_are_multiple_fds_opened (local->fd->inode, this))
+                goto out;
+
+        res = _gf_true;
+out:
+        return res;
+}
+
+
+void
+afr_delayed_changelog_post_op (xlator_t *this, call_frame_t *frame, fd_t *fd,
+                               call_stub_t *stub);
 
-                afr_pid_restore (frame);
+void
+afr_delayed_changelog_wake_up_cbk (void *data)
+{
+        fd_t           *fd = NULL;
+
+        fd = data;
+
+        afr_delayed_changelog_wake_up (THIS, fd);
+}
+
+
+/*
+  Check if the frame is destined to get optimized away
+  with changelog piggybacking
+*/
+static gf_boolean_t
+is_piggyback_post_op (call_frame_t *frame, fd_t *fd)
+{
+        afr_fd_ctx_t *fdctx = NULL;
+        afr_local_t *local = NULL;
+        gf_boolean_t piggyback = _gf_true;
+        afr_private_t *priv = NULL;
+        int i = 0;
 
-                local->transaction.fop (frame, this);
+        priv = frame->this->private;
+        local = frame->local;
+        fdctx = afr_fd_ctx_get (fd, frame->this);
+
+        LOCK(&fd->lock);
+        {
+                piggyback = _gf_true;
+
+                for (i = 0; i < priv->child_count; i++) {
+                        if (!local->transaction.pre_op[i])
+                                continue;
+                        if (fdctx->pre_op_piggyback[i]) {
+                                fdctx->pre_op_piggyback[i]--;
+                                local->transaction.postop_piggybacked[i] = 1;
+                        } else {
+                                /* For at least _one_ subvolume we cannot
+                                   piggyback on the changelog, and have to
+                                   perform a hard POST-OP and therefore fsync
+                                   if necesssary
+                                */
+                                piggyback = _gf_false;
+                                GF_ASSERT (fdctx->pre_op_done[i]);
+                                fdctx->pre_op_done[i]--;
+                        }
+                }
         }
+        UNLOCK(&fd->lock);
+
+        if (!afr_txn_nothing_failed (frame, frame->this)) {
+                /* something failed in this transaction,
+                   we will be performing a hard post-op
+                */
+                return _gf_false;
+        }
+
+        return piggyback;
+}
+
+
+/* SET operation */
+int
+afr_fd_report_unstable_write (xlator_t *this, fd_t *fd)
+{
+        afr_fd_ctx_t *fdctx = NULL;
+
+        fdctx = afr_fd_ctx_get (fd, this);
+
+        LOCK(&fd->lock);
+        {
+                fdctx->witnessed_unstable_write = _gf_true;
+        }
+        UNLOCK(&fd->lock);
 
         return 0;
 }
 
+/* TEST and CLEAR operation */
+gf_boolean_t
+afr_fd_has_witnessed_unstable_write (xlator_t *this, fd_t *fd)
+{
+        afr_fd_ctx_t *fdctx = NULL;
+        gf_boolean_t witness = _gf_false;
+
+	fdctx = afr_fd_ctx_get (fd, this);
+        if (!fdctx)
+                return _gf_true;
+
+        LOCK(&fd->lock);
+        {
+                if (fdctx->witnessed_unstable_write) {
+                        witness = _gf_true;
+                        fdctx->witnessed_unstable_write = _gf_false;
+                }
+        }
+        UNLOCK (&fd->lock);
+
+        return witness;
+}
+
 
 int
+afr_changelog_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, struct iatt *pre,
+                         struct iatt *post, dict_t *xdata)
+{
+        afr_private_t *priv = NULL;
+        int child_index = (long) cookie;
+        int call_count = -1;
+        afr_local_t *local = NULL;
+
+        priv = this->private;
+        local = frame->local;
+
+        if (afr_fop_failed (op_ret, op_errno)) {
+                /* Failure of fsync() is as good as failure of previous
+                   write(). So treat it like one.
+                */
+                gf_log (this->name, GF_LOG_WARNING,
+                        "fsync(%s) failed on subvolume %s. Transaction was %s",
+                        uuid_utoa (local->fd->inode->gfid),
+                        priv->children[child_index]->name,
+                        gf_fop_list[local->op]);
+
+                afr_transaction_fop_failed (frame, this, child_index);
+        }
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0)
+                afr_changelog_post_op_now (frame, this);
+
+        return 0;
+}
+
+
+int
+afr_changelog_fsync (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        int i = 0;
+        int call_count = 0;
+        afr_private_t *priv = NULL;
+ 	dict_t *xdata = NULL;
+ 	GF_UNUSED int ret = -1;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (!call_count) {
+                /* will go straight to unlock */
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+	xdata = dict_new();
+	if (xdata)
+		ret = dict_set_int32 (xdata, "batch-fsync", 1);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (!local->transaction.pre_op[i])
+                        continue;
+
+                STACK_WIND_COOKIE (frame, afr_changelog_fsync_cbk,
+                                (void *) (long) i, priv->children[i],
+                                priv->children[i]->fops->fsync, local->fd,
+                                1, xdata);
+                if (!--call_count)
+                        break;
+        }
+
+	if (xdata)
+		dict_unref (xdata);
+
+        return 0;
+}
+
+
+        int
+afr_changelog_post_op_safe (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t    *local = NULL;
+        afr_private_t  *priv = NULL;
+
+	local = frame->local;
+        priv = this->private;
+
+        if (!local->fd || local->transaction.type != AFR_DATA_TRANSACTION) {
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        if (is_piggyback_post_op (frame, local->fd)) {
+                /* just detected that this post-op is about to
+                   be optimized away as a new write() has
+                   already piggybacked on this frame's changelog.
+                   */
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        /* Calling afr_changelog_post_op_now() now will result in
+           issuing ->[f]xattrop().
+
+           Performing a hard POST-OP (->[f]xattrop() FOP) is a more
+           responsible operation that what it might appear on the surface.
+
+           The changelog of a file (in the xattr of the file on the server)
+           stores information (pending count) about the state of the file
+           on the OTHER server. This changelog is blindly trusted, and must
+           therefore be updated in such a way it remains trustworthy. This
+           implies that decrementing the pending count (essentially "clearing
+           the dirty flag") must be done STRICTLY after we are sure that the
+           operation on the other server has reached stable storage.
+
+           While the backend filesystem on that server will eventually flush
+           it to stable storage, we (being in userspace) have no mechanism
+           to get notified when the write became "stable".
+
+           This means we need take matter into our own hands and issue an
+           fsync() EVEN IF THE APPLICATION WAS PERFORMING UNSTABLE WRITES,
+           and get an acknowledgement for it. And we need to wait for the
+           fsync() acknowledgement before initiating the hard POST-OP.
+
+           However if the FD itself was opened in O_SYNC or O_DSYNC then
+           we are already guaranteed that the writes were made stable as
+           part of the FOP itself. The same holds true for NFS stable
+           writes which happen on an anonymous FD with O_DSYNC or O_SYNC
+           flag set in the writev() @flags param. For all other write types,
+           mark a flag in the fdctx whenever an unstable write is witnessed.
+           */
+
+        if (!afr_fd_has_witnessed_unstable_write (this, local->fd)) {
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        /* Check whether users want durability and perform fsync/post-op
+         * accordingly.
+         */
+        if (priv->ensure_durability) {
+                /* Time to fsync() */
+                afr_changelog_fsync (frame, this);
+        } else {
+                afr_changelog_post_op_now (frame, this);
+        }
+
+        return 0;
+}
+
+
+void
+afr_delayed_changelog_post_op (xlator_t *this, call_frame_t *frame, fd_t *fd,
+                               call_stub_t *stub)
+{
+	afr_fd_ctx_t      *fd_ctx = NULL;
+	call_frame_t      *prev_frame = NULL;
+	struct timespec    delta = {0, };
+	afr_private_t     *priv = NULL;
+	afr_local_t       *local = NULL;
+
+	priv = this->private;
+
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx)
+                goto out;
+
+	delta.tv_sec = priv->post_op_delay_secs;
+	delta.tv_nsec = 0;
+
+	pthread_mutex_lock (&fd_ctx->delay_lock);
+	{
+		prev_frame = fd_ctx->delay_frame;
+		fd_ctx->delay_frame = NULL;
+		if (fd_ctx->delay_timer)
+			gf_timer_call_cancel (this->ctx, fd_ctx->delay_timer);
+		fd_ctx->delay_timer = NULL;
+		if (!frame)
+			goto unlock;
+		fd_ctx->delay_timer = gf_timer_call_after (this->ctx, delta,
+							   afr_delayed_changelog_wake_up_cbk,
+							   fd);
+		fd_ctx->delay_frame = frame;
+	}
+unlock:
+	pthread_mutex_unlock (&fd_ctx->delay_lock);
+
+out:
+	if (prev_frame) {
+		local = prev_frame->local;
+		local->transaction.resume_stub = stub;
+		afr_changelog_post_op_safe (prev_frame, this);
+	} else if (stub) {
+		call_resume (stub);
+	}
+}
+
+
+void
+afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (is_afr_delayed_changelog_post_op_needed (frame, this))
+                afr_delayed_changelog_post_op (this, frame, local->fd, NULL);
+        else
+                afr_changelog_post_op_safe (frame, this);
+}
+
+
+
+/* Wake up the sleeping/delayed post-op, and also register
+   a stub to have it resumed after this transaction
+   completely finishes.
+
+   The @stub gets saved in @local and gets resumed in
+   afr_local_cleanup()
+   */
+void
+afr_delayed_changelog_wake_resume (xlator_t *this, fd_t *fd, call_stub_t *stub)
+{
+        afr_delayed_changelog_post_op (this, NULL, fd, stub);
+}
+
+
+void
+afr_delayed_changelog_wake_up (xlator_t *this, fd_t *fd)
+{
+        afr_delayed_changelog_post_op (this, NULL, fd, NULL);
+}
+
+
+        int
 afr_transaction_resume (call_frame_t *frame, xlator_t *this)
 {
         afr_internal_lock_t *int_lock = NULL;
@@ -1214,6 +1787,19 @@ afr_transaction_resume (call_frame_t *frame, xlator_t *this)
         int_lock = &local->internal_lock;
         priv     = this->private;
 
+        if (local->transaction.eager_lock_on) {
+                /* We don't need to retain "local" in the
+                   fd list anymore, writes to all subvols
+                   are finished by now */
+                LOCK (&local->fd->lock);
+                {
+                        list_del_init (&local->transaction.eager_locked);
+                }
+                UNLOCK (&local->fd->lock);
+        }
+
+        afr_restore_lk_owner (frame);
+
         if (__fop_changelog_needed (frame, this)) {
                 afr_changelog_post_op (frame, this);
         } else {
@@ -1234,7 +1820,8 @@ afr_transaction_resume (call_frame_t *frame, xlator_t *this)
  */
 
 void
-afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this, int child_index)
+afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
+                            int child_index)
 {
         afr_local_t *   local = NULL;
         afr_private_t * priv  = NULL;
@@ -1243,7 +1830,89 @@ afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this, int child_index
         priv  = this->private;
 
         __mark_child_dead (local->pending, priv->child_count,
-                           child_index, local->transaction.type);
+                        child_index, local->transaction.type);
+}
+
+
+
+        static gf_boolean_t
+afr_locals_overlap (afr_local_t *local1, afr_local_t *local2)
+{
+        uint64_t start1 = local1->transaction.start;
+        uint64_t start2 = local2->transaction.start;
+        uint64_t end1 = 0;
+        uint64_t end2 = 0;
+
+        if (local1->transaction.len)
+                end1 = start1 + local1->transaction.len - 1;
+        else
+                end1 = ULLONG_MAX;
+
+        if (local2->transaction.len)
+                end2 = start2 + local2->transaction.len - 1;
+        else
+                end2 = ULLONG_MAX;
+
+        return ((end1 >= start2) && (end2 >= start1));
+}
+
+void
+afr_transaction_eager_lock_init (afr_local_t *local, xlator_t *this)
+{
+        afr_private_t *priv = NULL;
+        afr_fd_ctx_t  *fdctx = NULL;
+        afr_local_t   *each = NULL;
+
+        priv = this->private;
+
+        if (!local->fd)
+                return;
+
+        if (local->transaction.type != AFR_DATA_TRANSACTION)
+                return;
+
+        if (!priv->eager_lock)
+                return;
+
+        fdctx = afr_fd_ctx_get (local->fd, this);
+        if (!fdctx)
+                return;
+
+        if (afr_are_multiple_fds_opened (local->fd->inode, this))
+                return;
+        /*
+         * Once full file lock is acquired in eager-lock phase, overlapping
+         * writes do not compete for inode-locks, instead are transferred to the
+         * next writes. Because of this overlapping writes are not ordered.
+         * This can cause inconsistencies in replication.
+         * Example:
+         * Two overlapping writes w1, w2 are sent in parallel on same fd
+         * in two threads t1, t2.
+         * Both threads can execute afr_writev_wind in the following manner.
+         * t1 winds w1 on brick-0
+         * t2 winds w2 on brick-0
+         * t2 winds w2 on brick-1
+         * t1 winds w1 on brick-1
+         *
+         * This check makes sure the locks are not transferred for
+         * overlapping writes.
+         */
+        LOCK (&local->fd->lock);
+        {
+                list_for_each_entry (each, &fdctx->eager_locked,
+                                     transaction.eager_locked) {
+                        if (afr_locals_overlap (each, local)) {
+                                local->transaction.eager_lock_on = _gf_false;
+                                goto unlock;
+                        }
+                }
+
+                local->transaction.eager_lock_on = _gf_true;
+                list_add_tail (&local->transaction.eager_locked,
+                               &fdctx->eager_locked);
+        }
+unlock:
+        UNLOCK (&local->fd->lock);
 }
 
 
@@ -1252,20 +1921,43 @@ afr_transaction (call_frame_t *frame, xlator_t *this, afr_transaction_type type)
 {
         afr_local_t *   local = NULL;
         afr_private_t * priv  = NULL;
+        fd_t            *fd   = NULL;
+        int             ret   = -1;
 
         local = frame->local;
         priv  = this->private;
 
-        afr_transaction_local_init (local, this);
-
         local->transaction.resume = afr_transaction_resume;
         local->transaction.type   = type;
 
+        ret = afr_transaction_local_init (local, this);
+        if (ret < 0)
+            goto out;
+
+        afr_transaction_eager_lock_init (local, this);
+
+        if (local->fd && local->transaction.eager_lock_on)
+                afr_set_lk_owner (frame, this, local->fd);
+        else
+                afr_set_lk_owner (frame, this, frame->root);
+
+        if (!local->transaction.eager_lock_on && local->loc.inode) {
+                fd = fd_lookup (local->loc.inode, frame->root->pid);
+                if (fd == NULL)
+                        fd = fd_lookup_anonymous (local->loc.inode);
+
+                if (fd) {
+                        afr_delayed_changelog_wake_up (this, fd);
+                        fd_unref (fd);
+                }
+        }
+
         if (afr_lock_server_count (priv, local->transaction.type) == 0) {
                 afr_internal_lock_finish (frame, this);
         } else {
                 afr_lock (frame, this);
         }
-
-        return 0;
+        ret = 0;
+out:
+        return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-transaction.h b/xlators/cluster/afr/src/afr-transaction.h
index f470f2697..fa626fd0d 100644
--- a/xlators/cluster/afr/src/afr-transaction.h
+++ b/xlators/cluster/afr/src/afr-transaction.h
@@ -1,25 +1,21 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __TRANSACTION_H__
 #define __TRANSACTION_H__
 
+typedef enum {
+        LOCAL_FIRST = 1,
+        LOCAL_LAST = 2
+} afr_xattrop_type_t;
+
 void
 afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
 			    int child_index);
@@ -27,11 +23,29 @@ afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
 int
 afr_lock_server_count (afr_private_t *priv, afr_transaction_type type);
 
+afr_inodelk_t*
+afr_get_inodelk (afr_internal_lock_t *int_lock, char *dom);
+
 int32_t
 afr_transaction (call_frame_t *frame, xlator_t *this, afr_transaction_type type);
 
 afr_fd_ctx_t *
 afr_fd_ctx_get (fd_t *fd, xlator_t *this);
 int
-afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending);
+afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending,
+                      int child, afr_xattrop_type_t op);
+void
+afr_set_delayed_post_op (call_frame_t *frame, xlator_t *this);
+
+void
+afr_delayed_changelog_wake_up (xlator_t *this, fd_t *fd);
+
+void
+__mark_all_success (int32_t *pending[], int child_count,
+                    afr_transaction_type type);
+gf_boolean_t
+afr_any_fops_failed (afr_local_t *local, afr_private_t *priv);
+
+gf_boolean_t
+afr_txn_nothing_failed (call_frame_t *frame, xlator_t *this);
 #endif /* __TRANSACTION_H__ */
diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index 89a8e87fb..c724eb2ae 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -116,6 +107,7 @@ reconfigure (xlator_t *this, dict_t *options)
 {
         afr_private_t *priv        = NULL;
         xlator_t      *read_subvol = NULL;
+        int            read_subvol_index = -1;
         int            ret         = -1;
         int            index       = -1;
         char          *qtype       = NULL;
@@ -158,6 +150,9 @@ reconfigure (xlator_t *this, dict_t *options)
 
         GF_OPTION_RECONF ("read-subvolume", read_subvol, options, xlator, out);
 
+        GF_OPTION_RECONF ("read-hash-mode", priv->hash_mode,
+                          options, uint32, out);
+
         if (read_subvol) {
                 index = xlator_subvolume_index (this, read_subvol);
                 if (index == -1) {
@@ -168,11 +163,37 @@ reconfigure (xlator_t *this, dict_t *options)
                 priv->read_child = index;
         }
 
+        GF_OPTION_RECONF ("read-subvolume-index",read_subvol_index, options,int32,out);
+
+        if (read_subvol_index >-1) {
+                index=read_subvol_index;
+                if (index >= priv->child_count) {
+                        gf_log (this->name, GF_LOG_ERROR, "%d not a subvolume-index",
+                                index);
+                        goto out;
+                }
+                priv->read_child = index;
+        }
+
         GF_OPTION_RECONF ("eager-lock", priv->eager_lock, options, bool, out);
         GF_OPTION_RECONF ("quorum-type", qtype, options, str, out);
         GF_OPTION_RECONF ("quorum-count", priv->quorum_count, options,
                           uint32, out);
         fix_quorum_options(this,priv,qtype);
+        GF_OPTION_RECONF ("heal-timeout", priv->shd.timeout, options,
+                          int32, out);
+
+	GF_OPTION_RECONF ("post-op-delay-secs", priv->post_op_delay_secs, options,
+			  uint32, out);
+
+        GF_OPTION_RECONF (AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size,
+                          options, size, out);
+        /* Reset this so we re-discover in case the topology changed.  */
+        GF_OPTION_RECONF ("readdir-failover", priv->readdir_failover, options,
+                          bool, out);
+        GF_OPTION_RECONF ("ensure-durability", priv->ensure_durability, options,
+                          bool, out);
+        priv->did_discovery = _gf_false;
 
         ret = 0;
 out:
@@ -200,6 +221,7 @@ init (xlator_t *this)
         int            ret         = -1;
         GF_UNUSED int  op_errno    = 0;
         xlator_t      *read_subvol = NULL;
+        int            read_subvol_index = -1;
         xlator_t      *fav_child   = NULL;
         char          *qtype       = NULL;
 
@@ -231,7 +253,6 @@ init (xlator_t *this)
 
         priv->child_count = child_count;
 
-
         priv->read_child = -1;
 
         GF_OPTION_INIT ("read-subvolume", read_subvol, xlator, out);
@@ -243,6 +264,18 @@ init (xlator_t *this)
                         goto out;
                 }
         }
+        GF_OPTION_INIT ("read-subvolume-index",read_subvol_index,int32,out);
+        if (read_subvol_index > -1) {
+                if (read_subvol_index >= priv->child_count) {
+                        gf_log (this->name, GF_LOG_ERROR, "%d not a subvolume-index",
+                                read_subvol_index);
+                        goto out;
+                }
+                priv->read_child = read_subvol_index;
+        }
+        GF_OPTION_INIT ("choose-local", priv->choose_local, bool, out);
+
+        GF_OPTION_INIT ("read-hash-mode", priv->hash_mode, uint32, out);
 
         priv->favorite_child = -1;
         GF_OPTION_INIT ("favorite-child", fav_child, xlator, out);
@@ -298,8 +331,15 @@ init (xlator_t *this)
         GF_OPTION_INIT ("eager-lock", priv->eager_lock, bool, out);
         GF_OPTION_INIT ("quorum-type", qtype, str, out);
         GF_OPTION_INIT ("quorum-count", priv->quorum_count, uint32, out);
+        GF_OPTION_INIT (AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size, size,
+                        out);
         fix_quorum_options(this,priv,qtype);
 
+	GF_OPTION_INIT ("post-op-delay-secs", priv->post_op_delay_secs, uint32, out);
+        GF_OPTION_INIT ("readdir-failover", priv->readdir_failover, bool, out);
+        GF_OPTION_INIT ("ensure-durability", priv->ensure_durability, bool,
+                        out);
+
         priv->wait_count = 1;
 
         priv->child_up = GF_CALLOC (sizeof (unsigned char), child_count,
@@ -340,8 +380,6 @@ init (xlator_t *this)
                                    AFR_XATTR_PREFIX,
                                    trav->xlator->name);
                 if (-1 == ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "asprintf failed to set pending key");
                         ret = -ENOMEM;
                         goto out;
                 }
@@ -350,6 +388,13 @@ init (xlator_t *this)
                 i++;
         }
 
+        ret = gf_asprintf (&priv->sh_domain, AFR_SH_DATA_DOMAIN_FMT,
+                           this->name);
+        if (-1 == ret) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
         priv->last_event = GF_CALLOC (child_count, sizeof (*priv->last_event),
                                       gf_afr_mt_int32_t);
         if (!priv->last_event) {
@@ -394,29 +439,30 @@ init (xlator_t *this)
         if (!priv->shd.timer)
                 goto out;
 
-        priv->shd.healed = eh_new (AFR_EH_HEALED_LIMIT, _gf_false);
+        priv->shd.healed = eh_new (AFR_EH_HEALED_LIMIT, _gf_false,
+                                   _destroy_shd_event_data);
         if (!priv->shd.healed)
                 goto out;
 
-        priv->shd.heal_failed = eh_new (AFR_EH_HEAL_FAIL_LIMIT, _gf_false);
+        priv->shd.heal_failed = eh_new (AFR_EH_HEAL_FAIL_LIMIT, _gf_false,
+                                        _destroy_shd_event_data);
         if (!priv->shd.heal_failed)
                 goto out;
 
-        priv->shd.split_brain = eh_new (AFR_EH_SPLIT_BRAIN_LIMIT, _gf_false);
+        priv->shd.split_brain = eh_new (AFR_EH_SPLIT_BRAIN_LIMIT, _gf_false,
+                                        _destroy_shd_event_data);
         if (!priv->shd.split_brain)
                 goto out;
 
-        priv->shd.sh_times = GF_CALLOC (priv->child_count,
-                                        sizeof (*priv->shd.sh_times),
-                                        gf_afr_mt_time_t);
-        if (!priv->shd.sh_times)
-                goto out;
-
         this->itable = inode_table_new (SHD_INODE_LRU_LIMIT, this);
         if (!this->itable)
                 goto out;
         priv->root_inode = inode_ref (this->itable->root);
-
+        GF_OPTION_INIT ("node-uuid", priv->shd.node_uuid, str, out);
+        GF_OPTION_INIT ("heal-timeout", priv->shd.timeout, int32, out);
+        ret = afr_initialise_statistics (this);
+        if (ret)
+                goto out;
         ret = 0;
 out:
         return ret;
@@ -451,6 +497,9 @@ struct xlator_fops fops = {
         .finodelk    = afr_finodelk,
         .entrylk     = afr_entrylk,
         .fentrylk    = afr_fentrylk,
+	.fallocate   = afr_fallocate,
+	.discard     = afr_discard,
+        .zerofill    = afr_zerofill,
 
         /* inode read */
         .access      = afr_access,
@@ -503,33 +552,79 @@ struct xlator_cbks cbks = {
 
 struct volume_options options[] = {
         { .key  = {"read-subvolume" },
-          .type = GF_OPTION_TYPE_XLATOR
+          .type = GF_OPTION_TYPE_XLATOR,
+          .description = "inode-read fops happen only on one of the bricks in "
+                         "replicate. Afr will prefer the one specified using "
+                         "this option if it is not stale. Option value must be "
+                         "one of the xlator names of the children. "
+                         "Ex: <volname>-client-0 till "
+                         "<volname>-client-<number-of-bricks - 1>"
+        },
+        { .key  = {"read-subvolume-index" },
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = "-1",
+          .description = "inode-read fops happen only on one of the bricks in "
+                         "replicate. AFR will prefer the one specified using "
+                         "this option if it is not stale. allowed options"
+                         " include -1 till replica-count - 1"
+        },
+        { .key = {"read-hash-mode" },
+          .type = GF_OPTION_TYPE_INT,
+          .min = 0,
+          .max = 2,
+          .default_value = "0",
+          .description = "inode-read fops happen only on one of the bricks in "
+                         "replicate. AFR will prefer the one computed using "
+                         "the method specified using this option"
+                         "0 = first responder, "
+                         "1 = hash by GFID of file (all clients use "
+                                                    "same subvolume), "
+                         "2 = hash by GFID of file and client PID",
+        },
+        { .key  = {"choose-local" },
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "true",
+          .description = "Choose a local subvolume(i.e. Brick) to read from if "
+                         "read-subvolume is not explicitly set.",
         },
         { .key  = {"favorite-child"},
-          .type = GF_OPTION_TYPE_XLATOR
+          .type = GF_OPTION_TYPE_XLATOR,
+          .description = "If a split-brain happens choose subvol/brick set by "
+                         "this option as source."
         },
         { .key  = {"background-self-heal-count"},
           .type = GF_OPTION_TYPE_INT,
           .min  = 0,
           .default_value = "16",
+          .validate = GF_OPT_VALIDATE_MIN,
+          .description = "This specifies the number of self-heals that can be "
+                         " performed in background without blocking the fop"
         },
         { .key  = {"data-self-heal"},
           .type = GF_OPTION_TYPE_STR,
-          .default_value = "",
           .value = {"1", "on", "yes", "true", "enable",
                     "0", "off", "no", "false", "disable",
                     "open"},
           .default_value = "on",
+          .description   = "Using this option we can enable/disable data "
+                           "self-heal on the file. \"open\" means data "
+                           "self-heal action will only be triggered by file "
+                           "open operations."
         },
         { .key  = {"data-self-heal-algorithm"},
           .type = GF_OPTION_TYPE_STR,
-          .default_value = "",
           .description   = "Select between \"full\", \"diff\". The "
                            "\"full\" algorithm copies the entire file from "
                            "source to sink. The \"diff\" algorithm copies to "
                            "sink only those blocks whose checksums don't match "
-                           "with those of source.",
-          .value = { "diff", "full", "" }
+                           "with those of source. If no option is configured "
+                           "the option is chosen dynamically as follows: "
+                           "If the file does not exist on one of the sinks "
+                           "or empty file exists or if the source file size is "
+                           "about the same as page size the entire file will "
+                           "be read and written i.e \"full\" algo, "
+                           "otherwise \"diff\" algo is chosen.",
+          .value = { "diff", "full"}
         },
         { .key  = {"data-self-heal-window-size"},
           .type = GF_OPTION_TYPE_INT,
@@ -542,26 +637,43 @@ struct volume_options options[] = {
         { .key  = {"metadata-self-heal"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Using this option we can enable/disable metadata "
+                         "i.e. Permissions, ownerships, xattrs self-heal on "
+                         "the file/directory."
         },
         { .key  = {"entry-self-heal"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Using this option we can enable/disable entry "
+                         "self-heal on the directory."
         },
         { .key  = {"data-change-log"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Data fops like write/truncate will not perform "
+                         "pre/post fop changelog operations in afr transaction "
+                         "if this option is disabled"
         },
         { .key  = {"metadata-change-log"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Metadata fops like setattr/setxattr will not perform "
+                         "pre/post fop changelog operations in afr transaction "
+                         "if this option is disabled"
         },
         { .key  = {"entry-change-log"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Entry fops like create/unlink will not perform "
+                         "pre/post fop changelog operations in afr transaction "
+                         "if this option is disabled"
         },
         { .key  = {"optimistic-change-log"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Entry/Metadata fops will not perform "
+                         "pre fop changelog operations in afr transaction "
+                         "if this option is enabled."
         },
         { .key  = {"strict-readdir"},
           .type = GF_OPTION_TYPE_BOOL,
@@ -570,26 +682,55 @@ struct volume_options options[] = {
         { .key = {"inodelk-trace"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "off",
+          .description = "Enabling this option logs inode lock/unlocks"
         },
         { .key = {"entrylk-trace"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "off",
+          .description = "Enabling this option logs entry lock/unlocks"
         },
         { .key = {"eager-lock"},
           .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
+          .default_value = "on",
+          .description = "Lock phase of a transaction has two sub-phases. "
+                         "First is an attempt to acquire locks in parallel by "
+                         "broadcasting non-blocking lock requests. If lock "
+                         "aquistion fails on any server, then the held locks "
+                         "are unlocked and revert to a blocking locked mode "
+                         "sequentially on one server after another.  If this "
+                         "option is enabled the initial broadcasting lock "
+                         "request attempt to acquire lock on the entire file. "
+                         "If this fails, we revert back to the sequential "
+                         "\"regional\" blocking lock as before. In the case "
+                         "where such an \"eager\" lock is granted in the "
+                         "non-blocking phase, it gives rise to an opportunity "
+                         "for optimization. i.e, if the next write transaction "
+                         "on the same FD arrives before the unlock phase of "
+                         "the first transaction, it \"takes over\" the full "
+                         "file lock. Similarly if yet another data transaction "
+                         "arrives before the unlock phase of the \"optimized\" "
+                         "transaction, that in turn \"takes over\" the lock as "
+                         "well. The actual unlock now happens at the end of "
+                         "the last \"optimzed\" transaction."
+
         },
         { .key = {"self-heal-daemon"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "off",
+          .description = "This option applies to only self-heal-daemon. "
+                         "Index directory crawl and automatic healing of files"
+                         "will not be performed if this option is turned off."
         },
         { .key = {"iam-self-heal-daemon"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "off",
+          .description = "This option differentiates if the replicate "
+                         "translator is running as part of self-heal-daemon "
+                         "or not."
         },
         { .key = {"quorum-type"},
           .type = GF_OPTION_TYPE_STR,
-          .value = { "none", "auto", "fixed", "" },
+          .value = { "none", "auto", "fixed"},
           .default_value = "none",
           .description = "If value is \"fixed\" only allow writes if "
                          "quorum-count bricks are present.  If value is "
@@ -606,5 +747,47 @@ struct volume_options options[] = {
                          "this many bricks or present.  Other quorum types "
                          "will OVERWRITE this value.",
         },
+        { .key  = {"node-uuid"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "Local glusterd uuid string, used in starting "
+                         "self-heal-daemon so that it can crawl only on "
+                         "local index directories.",
+        },
+        { .key  = {"heal-timeout"},
+          .type = GF_OPTION_TYPE_INT,
+          .min  = 60,
+          .max  = INT_MAX,
+          .default_value = "600",
+          .description = "time interval for checking the need to self-heal "
+                         "in self-heal-daemon"
+        },
+        { .key  = {"post-op-delay-secs"},
+          .type = GF_OPTION_TYPE_INT,
+          .min  = 0,
+          .max  = INT_MAX,
+          .default_value = "1",
+          .description = "Time interval induced artificially before "
+	                 "post-operation phase of the transaction to "
+                         "enhance overlap of adjacent write operations.",
+        },
+        { .key = {AFR_SH_READDIR_SIZE_KEY},
+          .type = GF_OPTION_TYPE_SIZET,
+          .description = "readdirp size for performing entry self-heal",
+          .min = 1024,
+          .max = 131072,
+          .default_value = "1KB",
+        },
+        { .key = {"readdir-failover"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .description = "readdir(p) will not failover if this option is off",
+          .default_value = "on",
+        },
+        { .key = {"ensure-durability"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .description = "Afr performs fsyncs for transactions if this "
+                         "option is on to make sure the changelogs/data is "
+                         "written to the disk",
+          .default_value = "on",
+        },
         { .key  = {NULL} },
 };
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index 09b1bf2a9..21064db58 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -36,6 +27,11 @@
 
 #define AFR_XATTR_PREFIX "trusted.afr"
 #define AFR_PATHINFO_HEADER "REPLICATE:"
+#define AFR_SH_READDIR_SIZE_KEY "self-heal-readdir-size"
+#define AFR_SH_DATA_DOMAIN_FMT "%s:self-heal"
+
+#define AFR_LOCKEE_COUNT_MAX    3
+#define AFR_DOM_COUNT_MAX    3
 
 struct _pump_private;
 
@@ -66,10 +62,8 @@ typedef enum {
         AFR_INODE_SET_READ_CTX = 1,
         AFR_INODE_RM_STALE_CHILDREN,
         AFR_INODE_SET_OPENDIR_DONE,
-        AFR_INODE_SET_SPLIT_BRAIN,
         AFR_INODE_GET_READ_CTX,
         AFR_INODE_GET_OPENDIR_DONE,
-        AFR_INODE_GET_SPLIT_BRAIN,
 } afr_inode_op_t;
 
 typedef struct afr_inode_params_ {
@@ -83,28 +77,41 @@ typedef struct afr_inode_params_ {
         } u;
 } afr_inode_params_t;
 
+typedef enum afr_spb_state {
+        DONT_KNOW,
+        SPB,
+        NO_SPB
+} afr_spb_state_t;
+
 typedef struct afr_inode_ctx_ {
         uint64_t masks;
         int32_t  *fresh_children;//increasing order of latency
+        afr_spb_state_t mdata_spb;
+        afr_spb_state_t data_spb;
+        uint32_t        open_fd_count;
 } afr_inode_ctx_t;
 
 typedef enum {
         NONE,
         INDEX,
+        INDEX_TO_BE_HEALED,
         FULL,
 } afr_crawl_type_t;
 
 typedef struct afr_self_heald_ {
-        gf_boolean_t     enabled;
-        gf_boolean_t     iamshd;
-        afr_crawl_type_t *pending;
-        gf_boolean_t     *inprogress;
-        afr_child_pos_t  *pos;
-        time_t           *sh_times;
-        gf_timer_t       **timer;
-        eh_t             *healed;
-        eh_t             *heal_failed;
-        eh_t             *split_brain;
+        gf_boolean_t            enabled;
+        gf_boolean_t            iamshd;
+        afr_crawl_type_t        *pending;
+        gf_boolean_t            *inprogress;
+        afr_child_pos_t         *pos;
+        gf_timer_t              **timer;
+        eh_t                    *healed;
+        eh_t                    *heal_failed;
+        eh_t                    *split_brain;
+        eh_t                    **statistics;
+        void                    **crawl_events;
+        char                    *node_uuid;
+        int                     timeout;
 } afr_self_heald_t;
 
 typedef struct _afr_private {
@@ -138,6 +145,7 @@ typedef struct _afr_private {
         gf_boolean_t entry_change_log;      /* on/off */
 
         int read_child;               /* read-subvolume */
+        unsigned int hash_mode;       /* for when read_child is not set */
         int favorite_child;  /* subvolume to be preferred in resolving
                                          split-brain cases */
 
@@ -158,14 +166,48 @@ typedef struct _afr_private {
         struct list_head saved_fds;   /* list of fds on which locks have succeeded */
         gf_boolean_t      optimistic_change_log;
         gf_boolean_t      eager_lock;
+	uint32_t          post_op_delay_secs;
         unsigned int      quorum_count;
 
         char                   vol_uuid[UUID_SIZE + 1];
         int32_t                *last_event;
         afr_self_heald_t       shd;
+        gf_boolean_t           choose_local;
+        gf_boolean_t           did_discovery;
+        gf_boolean_t           readdir_failover;
+        uint64_t               sh_readdir_size;
+        gf_boolean_t           ensure_durability;
+        char                   *sh_domain;
 } afr_private_t;
 
+typedef enum {
+        AFR_SELF_HEAL_NOT_ATTEMPTED,
+        AFR_SELF_HEAL_STARTED,
+        AFR_SELF_HEAL_FAILED,
+        AFR_SELF_HEAL_SYNC_BEGIN,
+} afr_self_heal_status;
+
 typedef struct {
+        afr_self_heal_status gfid_or_missing_entry_self_heal;
+        afr_self_heal_status metadata_self_heal;
+        afr_self_heal_status data_self_heal;
+        afr_self_heal_status entry_self_heal;
+} afr_sh_status_for_all_type;
+
+typedef enum {
+        AFR_SELF_HEAL_ENTRY,
+        AFR_SELF_HEAL_METADATA,
+        AFR_SELF_HEAL_DATA,
+        AFR_SELF_HEAL_GFID_OR_MISSING_ENTRY,
+        AFR_SELF_HEAL_INVALID = -1,
+} afr_self_heal_type;
+
+typedef enum {
+        AFR_CHECK_ALL,
+        AFR_CHECK_SPECIFIC,
+} afr_sh_fail_check_type;
+
+struct afr_self_heal_ {
         /* External interface: These are variables (some optional) that
            are set by whoever has triggered self-heal */
 
@@ -174,6 +216,8 @@ typedef struct {
         gf_boolean_t do_entry_self_heal;
         gf_boolean_t do_gfid_self_heal;
         gf_boolean_t do_missing_entry_self_heal;
+        gf_boolean_t force_confirm_spb; /* Check for split-brains even when
+                                           self-heal is turned off */
 
         gf_boolean_t forced_merge;        /* Is this a self-heal triggered to
                                              forcibly merge the directories? */
@@ -191,7 +235,7 @@ typedef struct {
            background, this function will be called as soon as possible. */
 
         int (*unwind) (call_frame_t *frame, xlator_t *this, int32_t op_ret,
-                       int32_t op_errno);
+                       int32_t op_errno, int32_t sh_failed);
 
         /* End of external interface members */
 
@@ -240,9 +284,10 @@ typedef struct {
         const char *linkname;
         gf_boolean_t entries_skipped;
 
-        int   op_failed;
-
+        gf_boolean_t actual_sh_started;
+        gf_boolean_t sync_done;
         gf_boolean_t data_lock_held;
+        gf_boolean_t sh_dom_lock_held;
         gf_boolean_t eof_reached;
         fd_t  *healing_fd;
         int   file_has_holes;
@@ -253,16 +298,22 @@ typedef struct {
         uint8_t *checksum;
         afr_post_remove_call_t post_remove_call;
 
-        loc_t parent_loc;
+        char    *data_sh_info;
+        char    *metadata_sh_info;
 
+        loc_t parent_loc;
         call_frame_t *orig_frame;
         call_frame_t *old_loop_frame;
         gf_boolean_t unwound;
 
         afr_sh_algo_private_t *private;
+        afr_sh_status_for_all_type  afr_all_sh_status;
+        afr_self_heal_type       sh_type_in_action;
 
+        struct afr_sh_algorithm  *algo;
         afr_lock_cbk_t data_lock_success_handler;
         afr_lock_cbk_t data_lock_failure_handler;
+	gf_boolean_t data_lock_block;
         int (*completion_cbk) (call_frame_t *frame, xlator_t *this);
         int (*sh_data_algo_start) (call_frame_t *frame, xlator_t *this);
         int (*algo_completion_cbk) (call_frame_t *frame, xlator_t *this);
@@ -270,7 +321,9 @@ typedef struct {
         void (*gfid_sh_success_cbk) (call_frame_t *sh_frame, xlator_t *this);
 
         call_frame_t *sh_frame;
-} afr_self_heal_t;
+};
+
+typedef struct afr_self_heal_ afr_self_heal_t;
 
 typedef enum {
         AFR_DATA_TRANSACTION,          /* truncate, write, ... */
@@ -332,11 +385,31 @@ afr_index_for_transaction_type (afr_transaction_type type)
         return -1;  /* make gcc happy */
 }
 
+typedef struct {
+        loc_t                   loc;
+        char                    *basename;
+        unsigned char           *locked_nodes;
+        int                     locked_count;
+
+} afr_entry_lockee_t;
+
+int
+afr_entry_lockee_cmp (const void *l1, const void *l2);
+
+typedef struct {
+        char    *domain; /* Domain on which inodelk is taken */
+        struct gf_flock flock;
+        unsigned char *locked_nodes;
+        int32_t lock_count;
+} afr_inodelk_t;
 
 typedef struct {
         loc_t *lk_loc;
-        struct gf_flock lk_flock;
 
+        int                     lockee_count;
+        afr_entry_lockee_t      lockee[AFR_LOCKEE_COUNT_MAX];
+
+        afr_inodelk_t       inodelk[AFR_DOM_COUNT_MAX];
         const char *lk_basename;
         const char *lower_basename;
         const char *higher_basename;
@@ -345,23 +418,22 @@ typedef struct {
 
         unsigned char *locked_nodes;
         unsigned char *lower_locked_nodes;
-        unsigned char *inode_locked_nodes;
-        unsigned char *entry_locked_nodes;
 
         selfheal_lk_type_t selfheal_lk_type;
         transaction_lk_type_t transaction_lk_type;
 
         int32_t lock_count;
-        int32_t inodelk_lock_count;
         int32_t entrylk_lock_count;
 
         uint64_t lock_number;
         int32_t lk_call_count;
         int32_t lk_expected_count;
+        int32_t lk_attempted_count;
 
         int32_t lock_op_ret;
         int32_t lock_op_errno;
         afr_lock_cbk_t lock_cbk;
+        char *domain; /* Domain on which inode/entry lock/unlock in progress.*/
 } afr_internal_lock_t;
 
 typedef struct _afr_locked_fd {
@@ -369,21 +441,29 @@ typedef struct _afr_locked_fd {
         struct list_head list;
 } afr_locked_fd_t;
 
+struct afr_reply {
+	int	valid;
+	int32_t	op_ret;
+	int32_t	op_errno;
+};
+
 typedef struct _afr_local {
         int     uid;
         int     gid;
         unsigned int call_count;
         unsigned int success_count;
         unsigned int enoent_count;
+        uint32_t     open_fd_count;
+        gf_boolean_t update_open_fd_count;
 
 
-        unsigned int govinda_gOvinda;
+        unsigned int unhealable;
 
         unsigned int read_child_index;
         unsigned char read_child_returned;
         unsigned int first_up_child;
 
-        pid_t saved_pid;
+	gf_lkowner_t  saved_lk_owner;
 
         int32_t op_ret;
         int32_t op_errno;
@@ -394,7 +474,6 @@ typedef struct _afr_local {
         loc_t newloc;
 
         fd_t *fd;
-        unsigned char *fd_open_on;
 
         glusterfs_fop_t fop;
 
@@ -416,13 +495,25 @@ typedef struct _afr_local {
 
         dict_t  *dict;
         int      optimistic_change_log;
+	gf_boolean_t      delayed_post_op;
 
-        gf_boolean_t    fop_paused;
-        int (*fop_call_continue) (call_frame_t *frame, xlator_t *this);
 
-        /*
-          This struct contains the arguments for the "continuation"
-          (scheme-like) of fops
+	/* Is the current writev() going to perform a stable write?
+	   i.e, is fd->flags or @flags writev param have O_SYNC or
+	   O_DSYNC?
+	*/
+        gf_boolean_t      stable_write;
+
+        /* This write appended to the file. Nnot necessarily O_APPEND,
+           just means the offset of write was at the end of file.
+        */
+        gf_boolean_t      append_write;
+
+        int allow_sh_for_running_transaction;
+
+
+        /* This struct contains the arguments for the "continuation"
+           (scheme-like) of fops
         */
 
         int   op;
@@ -433,6 +524,7 @@ typedef struct _afr_local {
                 } statfs;
 
                 struct {
+                        uint32_t parent_entrylk;
                         uuid_t  gfid_req;
                         inode_t *inode;
                         struct iatt buf;
@@ -444,12 +536,13 @@ typedef struct _afr_local {
                         int32_t read_child;
                         int32_t *sources;
                         int32_t *success_children;
+                        int32_t **pending_matrix;
                         gf_boolean_t fresh_lookup;
+                        gf_boolean_t possible_spb;
                 } lookup;
 
                 struct {
                         int32_t flags;
-                        int32_t wbflags;
                 } open;
 
                 struct {
@@ -516,7 +609,9 @@ typedef struct _afr_local {
                 struct {
                         struct iatt prebuf;
                         struct iatt postbuf;
+                } inode_wfop; //common structure for all inode-write-fops
 
+                struct {
                         int32_t op_ret;
 
                         struct iovec *vector;
@@ -527,34 +622,21 @@ typedef struct _afr_local {
                 } writev;
 
                 struct {
-                        struct iatt prebuf;
-                        struct iatt postbuf;
-                } fsync;
-
-                struct {
                         off_t offset;
-                        struct iatt prebuf;
-                        struct iatt postbuf;
                 } truncate;
 
                 struct {
                         off_t offset;
-                        struct iatt prebuf;
-                        struct iatt postbuf;
                 } ftruncate;
 
                 struct {
                         struct iatt in_buf;
                         int32_t valid;
-                        struct iatt preop_buf;
-                        struct iatt postop_buf;
                 } setattr;
 
                 struct {
                         struct iatt in_buf;
                         int32_t valid;
-                        struct iatt preop_buf;
-                        struct iatt postop_buf;
                 } fsetattr;
 
                 struct {
@@ -582,90 +664,66 @@ typedef struct _afr_local {
                 /* dir write */
 
                 struct {
-                        fd_t *fd;
-                        dict_t *params;
-                        int32_t flags;
-                        mode_t mode;
                         inode_t *inode;
                         struct iatt buf;
                         struct iatt preparent;
                         struct iatt postparent;
-                        struct iatt read_child_buf;
+                        struct iatt prenewparent;
+                        struct iatt postnewparent;
+                } dir_fop; //common structure for all dir fops
+
+                struct {
+                        fd_t *fd;
+                        dict_t *params;
+                        int32_t flags;
+                        mode_t mode;
                 } create;
 
                 struct {
                         dev_t dev;
                         mode_t mode;
                         dict_t *params;
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                        struct iatt read_child_buf;
                 } mknod;
 
                 struct {
                         int32_t mode;
                         dict_t *params;
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt read_child_buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
                 } mkdir;
 
                 struct {
-                        int32_t op_ret;
-                        int32_t op_errno;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                } unlink;
-
-                struct {
-                        int   flags;
-                        int32_t op_ret;
-                        int32_t op_errno;
-                        struct iatt preparent;
-                        struct iatt postparent;
+                        int flags;
                 } rmdir;
 
                 struct {
-                        struct iatt buf;
-                        struct iatt read_child_buf;
-                        struct iatt preoldparent;
-                        struct iatt prenewparent;
-                        struct iatt postoldparent;
-                        struct iatt postnewparent;
-                } rename;
-
-                struct {
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt read_child_buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                } link;
-
-                struct {
-                        inode_t *inode;
                         dict_t *params;
-                        struct iatt buf;
-                        struct iatt read_child_buf;
                         char *linkpath;
-                        struct iatt preparent;
-                        struct iatt postparent;
                 } symlink;
 
+		struct {
+			int32_t mode;
+			off_t offset;
+			size_t len;
+		} fallocate;
+
+		struct {
+			off_t offset;
+			size_t len;
+		} discard;
+
                 struct {
-                        int32_t flags;
-                        dir_entry_t *entries;
-                        int32_t count;
-                } setdents;
+                        off_t offset;
+                        size_t len;
+                        struct iatt prebuf;
+                        struct iatt postbuf;
+                } zerofill;
+
+
         } cont;
 
         struct {
                 off_t start, len;
 
+                gf_boolean_t    eager_lock_on;
                 int *eager_lock;
 
                 char *basename;
@@ -676,12 +734,18 @@ typedef struct _afr_local {
 
                 afr_transaction_type type;
 
-                int success_count;
-                int erase_pending;
-                int failure_count;
+		/* pre-compute the post piggyback status before
+		   entering POST-OP phase
+		*/
+		int              *postop_piggybacked;
 
-                int last_tried;
-                int32_t *child_errno;
+		/* stub to resume on destruction
+		   of the transaction frame */
+		call_stub_t      *resume_stub;
+
+		struct list_head  eager_locked;
+
+                int32_t         **txn_changelog;//changelog after pre+post ops
                 unsigned char   *pre_op;
 
                 call_frame_t *main_frame;
@@ -700,6 +764,15 @@ typedef struct _afr_local {
         afr_self_heal_t self_heal;
 
         struct marker_str     marker;
+
+        /* extra data for fops */
+        dict_t         *xdata_req;
+        dict_t         *xdata_rsp;
+
+        mode_t          umask;
+        int             xflag;
+        gf_boolean_t    do_discovery;
+	struct afr_reply *replies;
 } afr_local_t;
 
 typedef enum {
@@ -709,11 +782,6 @@ typedef enum {
 } afr_fd_open_status_t;
 
 typedef struct {
-        struct list_head call_list;
-        call_frame_t    *frame;
-} afr_fd_paused_call_t;
-
-typedef struct {
         unsigned int *pre_op_done;
         afr_fd_open_status_t *opened_on; /* which subvolumes the fd is open on */
         unsigned int *pre_op_piggyback;
@@ -722,7 +790,6 @@ typedef struct {
         unsigned int *lock_acquired;
 
         int flags;
-        int32_t wbflags;
         uint64_t up_count;   /* number of CHILD_UPs this fd has seen */
         uint64_t down_count; /* number of CHILD_DOWNs this fd has seen */
 
@@ -733,7 +800,20 @@ typedef struct {
         struct list_head entries; /* needed for readdir failover */
 
         unsigned char *locked_on; /* which subvolumes locks have been successful */
-	struct list_head  paused_calls; /* queued calls while fix_open happens  */
+
+	/* used for delayed-post-op optimization */
+	pthread_mutex_t    delay_lock;
+	gf_timer_t        *delay_timer;
+	call_frame_t      *delay_frame;
+        int               call_child;
+
+	/* set if any write on this fd was a non stable write
+	   (i.e, without O_SYNC or O_DSYNC)
+	*/
+	gf_boolean_t      witnessed_unstable_write;
+
+	/* list of frames currently in progress */
+	struct list_head  eager_locked;
 } afr_fd_ctx_t;
 
 
@@ -769,6 +849,13 @@ int32_t
 afr_notify (xlator_t *this, int32_t event, void *data, void *data2);
 
 int
+afr_init_entry_lockee (afr_entry_lockee_t *lockee, afr_local_t *local,
+                       loc_t *loc, char *basename, int child_count);
+
+void
+afr_entry_lockee_cleanup (afr_internal_lock_t *int_lock);
+
+int
 afr_attempt_lock_recovery (xlator_t *this, int32_t child_index);
 
 int
@@ -803,8 +890,8 @@ afr_blocking_lock (call_frame_t *frame, xlator_t *this);
 int
 afr_internal_lock_finish (call_frame_t *frame, xlator_t *this);
 
-void
-afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src,
+int
+afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src, char *dom,
                           unsigned int child_count);
 
 int pump_start (call_frame_t *frame, xlator_t *this);
@@ -854,11 +941,12 @@ gf_boolean_t
 afr_is_split_brain (xlator_t *this, inode_t *inode);
 
 void
-afr_set_split_brain (xlator_t *this, inode_t *inode, gf_boolean_t set);
+afr_set_split_brain (xlator_t *this, inode_t *inode, afr_spb_state_t mdata_spb,
+                     afr_spb_state_t data_spb);
 
 int
 afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, int32_t wbflags);
+          fd_t *fd, dict_t *xdata);
 
 void
 afr_set_opendir_done (xlator_t *this, inode_t *inode);
@@ -941,12 +1029,13 @@ afr_first_up_child (unsigned char *child_up, size_t child_count);
 int
 afr_select_read_child_from_policy (int32_t *fresh_children, int32_t child_count,
                                    int32_t prev_read_child,
-                                   int32_t config_read_child, int32_t *sources);
+                                   int32_t config_read_child, int32_t *sources,
+                                   unsigned int hmode, uuid_t gfid);
 
 void
 afr_set_read_ctx_from_policy (xlator_t *this, inode_t *inode,
                               int32_t *fresh_children, int32_t prev_read_child,
-                              int32_t config_read_child);
+                              int32_t config_read_child, uuid_t gfid);
 
 int32_t
 afr_get_call_child (xlator_t *this, unsigned char *child_up, int32_t read_child,
@@ -968,8 +1057,9 @@ afr_children_rm_child (int32_t *children, int32_t child,
                              int32_t child_count);
 void
 afr_reset_children (int32_t *children, int32_t child_count);
-gf_boolean_t
-afr_error_more_important (int32_t old_errno, int32_t new_errno);
+int32_t
+afr_most_important_error(int32_t old_errno, int32_t new_errno,
+			 gf_boolean_t eio);
 int
 afr_errno_count (int32_t *children, int *child_errno,
                  unsigned int child_count, int32_t op_errno);
@@ -1010,12 +1100,13 @@ afr_launch_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode,
                       void (*gfid_sh_success_cbk) (call_frame_t *sh_frame,
                                                    xlator_t *this),
                       int (*unwind) (call_frame_t *frame, xlator_t *this,
-                                     int32_t op_ret, int32_t op_errno));
-int
-afr_fix_open (call_frame_t *frame, xlator_t *this, afr_fd_ctx_t *fd_ctx,
-              int need_open_count, int *need_open);
-int
-afr_open_fd_fix (call_frame_t *frame, xlator_t *this, gf_boolean_t pause_fop);
+                                     int32_t op_ret, int32_t op_errno,
+                                     int32_t sh_failed));
+void
+afr_fix_open (xlator_t *this, fd_t *fd, size_t need_open_count, int *need_open);
+
+void
+afr_open_fd_fix (fd_t *fd, xlator_t *this);
 int
 afr_set_elem_count_get (unsigned char *elems, int child_count);
 
@@ -1032,7 +1123,7 @@ void
 afr_set_low_priority (call_frame_t *frame);
 int
 afr_child_fd_ctx_set (xlator_t *this, fd_t *fd, int32_t child,
-                      int flags, int32_t wb_flags);
+                      int flags);
 
 gf_boolean_t
 afr_have_quorum (char *logname, afr_private_t *priv);
@@ -1042,6 +1133,23 @@ afr_matrix_cleanup (int32_t **pending, unsigned int m);
 
 int32_t**
 afr_matrix_create (unsigned int m, unsigned int n);
+
+gf_boolean_t
+afr_is_errno_set (int *child_errno, int child);
+
+gf_boolean_t
+afr_is_errno_unset (int *child_errno, int child);
+
+gf_boolean_t
+afr_is_fd_fixable (fd_t *fd);
+
+void
+afr_prepare_new_entry_pending_matrix (int32_t **pending,
+                                      gf_boolean_t (*is_pending) (int *, int),
+                                      int *ctx, struct iatt *buf,
+                                      unsigned int child_count);
+void
+afr_xattr_array_destroy (dict_t **xattr, unsigned int child_count);
 /*
  * Special value indicating we should use the "auto" quorum method instead of
  * a fixed value (including zero to turn off quorum enforcement).
@@ -1061,4 +1169,44 @@ afr_matrix_create (unsigned int m, unsigned int n);
         }                                                                \
 } while (0);
 
+
+#define AFR_SBRAIN_MSG "Failed on %s as split-brain is seen. Returning EIO."
+
+#define AFR_SBRAIN_CHECK_FD(fd, label) do {                              \
+        if (fd->inode && afr_is_split_brain (this, fd->inode)) {        \
+                op_errno = EIO;                                         \
+                gf_log (this->name, GF_LOG_WARNING,                     \
+                        AFR_SBRAIN_MSG ,uuid_utoa (fd->inode->gfid));   \
+                goto label;                                             \
+        }                                                               \
+} while (0)
+
+#define AFR_SBRAIN_CHECK_LOC(loc, label) do {                           \
+        if (loc->inode && afr_is_split_brain (this, loc->inode)) {      \
+                op_errno = EIO;                                         \
+                loc_path (loc, NULL);                                   \
+                gf_log (this->name, GF_LOG_WARNING,                     \
+                        AFR_SBRAIN_MSG , loc->path);                    \
+                goto label;                                             \
+        }                                                               \
+} while (0)
+
+int
+afr_fd_report_unstable_write (xlator_t *this, fd_t *fd);
+
+gf_boolean_t
+afr_fd_has_witnessed_unstable_write (xlator_t *this, fd_t *fd);
+
+void
+afr_delayed_changelog_wake_resume (xlator_t *this, fd_t *fd, call_stub_t *stub);
+
+int
+afr_inodelk_init (afr_inodelk_t *lk, char *dom, size_t child_count);
+
+void
+afr_handle_open_fd_count (call_frame_t *frame, xlator_t *this);
+
+afr_inode_ctx_t*
+afr_inode_ctx_get (inode_t *inode, xlator_t *this);
+
 #endif /* __AFR_H__ */
diff --git a/xlators/cluster/afr/src/pump.c b/xlators/cluster/afr/src/pump.c
index 21d10aff2..a7f72fb30 100644
--- a/xlators/cluster/afr/src/pump.c
+++ b/xlators/cluster/afr/src/pump.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <unistd.h>
@@ -437,7 +428,7 @@ gf_pump_traverse_directory (loc_t *loc)
                                                     gf_pump_traverse_directory (&entry_loc);
                                             }
                                     }
-                            }
+                        }
                 }
 
                 gf_dirent_free (&entries);
@@ -448,6 +439,10 @@ gf_pump_traverse_directory (loc_t *loc)
 
         }
 
+        ret = syncop_close (fd);
+        if (ret < 0)
+                gf_log (this->name, GF_LOG_DEBUG, "closing the fd failed");
+
         if (is_directory_empty && IS_ROOT_PATH (loc->path)) {
                pump_change_state (this, PUMP_STATE_RUNNING);
                gf_log (this->name, GF_LOG_INFO, "Empty source brick. "
@@ -487,7 +482,7 @@ pump_update_resume_path (xlator_t *this)
 
 static int32_t
 pump_xattr_cleaner (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_private_t  *priv      = NULL;
         loc_t           loc       = {0};
@@ -753,7 +748,7 @@ pump_cmd_start_setxattr_cbk (call_frame_t *frame,
                              void *cookie,
                              xlator_t *this,
                              int32_t op_ret,
-                             int32_t op_errno)
+                             int32_t op_errno, dict_t *xdata)
 
 {
         call_frame_t *prev = NULL;
@@ -846,7 +841,7 @@ pump_initiate_sink_connect (call_frame_t *frame, xlator_t *this)
 		    PUMP_SINK_CHILD(this)->fops->setxattr,
 		    &loc,
 		    dict,
-		    0);
+		    0, NULL);
 
         ret = 0;
 
@@ -880,7 +875,7 @@ pump_cmd_start_getxattr_cbk (call_frame_t *frame,
                              xlator_t *this,
                              int32_t op_ret,
                              int32_t op_errno,
-                             dict_t *dict)
+                             dict_t *dict, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         char *path = NULL;
@@ -947,6 +942,7 @@ pump_execute_status (call_frame_t *frame, xlator_t *this)
         uint64_t number_files = 0;
 
         char filename[PATH_MAX];
+        char summary[PATH_MAX+256];
         char *dict_str = NULL;
 
         int32_t op_ret = 0;
@@ -975,12 +971,15 @@ pump_execute_status (call_frame_t *frame, xlator_t *this)
         }
 
         if (pump_priv->pump_finished) {
-        snprintf (dict_str, PATH_MAX + 256, "Number of files migrated = %"PRIu64"        Migration complete ",
-                  number_files);
+                snprintf (summary, PATH_MAX+256,
+                          "no_of_files=%"PRIu64, number_files);
         } else {
-        snprintf (dict_str, PATH_MAX + 256, "Number of files migrated = %"PRIu64"       Current file= %s ",
-                  number_files, filename);
+                snprintf (summary, PATH_MAX+256,
+                          "no_of_files=%"PRIu64":current_file=%s",
+                          number_files, filename);
         }
+        snprintf (dict_str, PATH_MAX+256, "status=%d:%s",
+                  (pump_priv->pump_finished)?1:0, summary);
 
         dict = dict_new ();
 
@@ -996,13 +995,12 @@ pump_execute_status (call_frame_t *frame, xlator_t *this)
 
 out:
 
-        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, NULL);
 
         if (dict)
                 dict_unref (dict);
 
-        if (dict_str)
-                GF_FREE (dict_str);
+        GF_FREE (dict_str);
 
         return 0;
 }
@@ -1051,7 +1049,7 @@ pump_execute_start (call_frame_t *frame, xlator_t *this)
 		    PUMP_SOURCE_CHILD(this),
 		    PUMP_SOURCE_CHILD(this)->fops->getxattr,
 		    &loc,
-		    PUMP_PATH);
+		    PUMP_PATH, NULL);
 
         ret = 0;
 
@@ -1069,7 +1067,7 @@ static int
 pump_cleanup_helper (void *data) {
         call_frame_t *frame = data;
 
-        pump_xattr_cleaner (frame, 0, frame->this, 0, 0);
+        pump_xattr_cleaner (frame, 0, frame->this, 0, 0, NULL);
 
         return 0;
 }
@@ -1151,7 +1149,7 @@ pump_execute_abort (call_frame_t *frame, xlator_t *this)
         } else {
                 pump_priv->cleaner = fop_setxattr_cbk_stub (frame,
                                                             pump_xattr_cleaner,
-                                                            0, 0);
+                                                            0, 0, NULL);
         }
 
         return 0;
@@ -1282,7 +1280,7 @@ struct _xattr_key {
         struct list_head list;
 };
 
-static void
+static int
 __gather_xattr_keys (dict_t *dict, char *key, data_t *value,
                      void *data)
 {
@@ -1294,13 +1292,14 @@ __gather_xattr_keys (dict_t *dict, char *key, data_t *value,
 
                 xkey = GF_CALLOC (1, sizeof (*xkey), gf_afr_mt_xattr_key);
                 if (!xkey)
-                        return;
+                        return -1;
 
                 xkey->key = key;
                 INIT_LIST_HEAD (&xkey->list);
 
                 list_add_tail (&xkey->list, list);
         }
+        return 0;
 }
 
 static void
@@ -1328,7 +1327,7 @@ __filter_xattrs (dict_t *dict)
 int32_t
 pump_getxattr_cbk (call_frame_t *frame, void *cookie,
 		  xlator_t *this, int32_t op_ret, int32_t op_errno,
-		  dict_t *dict)
+		  dict_t *dict, dict_t *xdata)
 {
 	afr_private_t   *priv           = NULL;
 	afr_local_t     *local          = NULL;
@@ -1363,7 +1362,7 @@ pump_getxattr_cbk (call_frame_t *frame, void *cookie,
 				   children[next_call_child],
 				   children[next_call_child]->fops->getxattr,
 				   &local->loc,
-				   local->cont.getxattr.name);
+				   local->cont.getxattr.name, NULL);
 	}
 
 out:
@@ -1371,7 +1370,7 @@ out:
                 if (op_ret >= 0 && dict)
                         __filter_xattrs (dict);
 
-		AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+		AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, NULL);
 	}
 
 	return 0;
@@ -1379,7 +1378,7 @@ out:
 
 int32_t
 pump_getxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *name)
+	      loc_t *loc, const char *name, dict_t *xdata)
 {
 	afr_private_t *   priv       = NULL;
 	xlator_t **       children   = NULL;
@@ -1402,7 +1401,7 @@ pump_getxattr (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_getxattr_cbk,
                             FIRST_CHILD (this),
                             (FIRST_CHILD (this))->fops->getxattr,
-                            loc, name);
+                            loc, name, xdata);
                 return 0;
         }
 
@@ -1456,12 +1455,12 @@ pump_getxattr (call_frame_t *frame, xlator_t *this,
 	STACK_WIND_COOKIE (frame, pump_getxattr_cbk,
 			   (void *) (long) call_child,
 			   children[call_child], children[call_child]->fops->getxattr,
-			   loc, name);
+			   loc, name, xdata);
 
 	ret = 0;
 out:
 	if (ret < 0)
-		AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL);
+		AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
 	return 0;
 }
 
@@ -1483,14 +1482,14 @@ afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
 
 	if (main_frame) {
 		AFR_STACK_UNWIND (setxattr, main_frame,
-                                  local->op_ret, local->op_errno);
+                                  local->op_ret, local->op_errno, NULL);
 	}
 	return 0;
 }
 
 static int
 afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno)
+		       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
 	afr_local_t *   local = NULL;
 	afr_private_t * priv  = NULL;
@@ -1559,7 +1558,7 @@ afr_setxattr_wind (call_frame_t *frame, xlator_t *this)
 					   priv->children[i]->fops->setxattr,
 					   &local->loc,
 					   local->cont.setxattr.dict,
-					   local->cont.setxattr.flags);
+					   local->cont.setxattr.flags, NULL);
 
 			if (!--call_count)
 				break;
@@ -1587,11 +1586,9 @@ pump_setxattr_cbk (call_frame_t *frame,
 		      void *cookie,
 		      xlator_t *this,
 		      int32_t op_ret,
-		      int32_t op_errno)
+		      int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno);
+	AFR_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
 	return 0;
 }
 
@@ -1609,12 +1606,10 @@ pump_command_reply (call_frame_t *frame, xlator_t *this)
                 gf_log (this->name, GF_LOG_INFO,
                         "Command succeeded");
 
-        dict_unref (local->dict);
-
         AFR_STACK_UNWIND (setxattr,
                           frame,
                           local->op_ret,
-                          local->op_errno);
+                          local->op_errno, NULL);
 
         return 0;
 }
@@ -1651,12 +1646,11 @@ pump_parse_command (call_frame_t *frame, xlator_t *this,
 
 int
 pump_setxattr (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, dict_t *dict, int32_t flags)
+               loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata)
 {
 	afr_private_t * priv  = NULL;
 	afr_local_t   * local = NULL;
 	call_frame_t   *transaction_frame = NULL;
-        data_pair_t   * trav  = NULL;
 	int ret = -1;
 	int op_errno = 0;
 
@@ -1665,14 +1659,14 @@ pump_setxattr (call_frame_t *frame, xlator_t *this,
 	VALIDATE_OR_GOTO (this->private, out);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.pump*", dict,
-                                   trav, op_errno, out);
+                                   op_errno, out);
 
 	priv = this->private;
         if (!priv->use_afr_in_pump) {
                 STACK_WIND (frame, default_setxattr_cbk,
                             FIRST_CHILD (this),
                             (FIRST_CHILD (this))->fops->setxattr,
-                            loc, dict, flags);
+                            loc, dict, flags, xdata);
                 return 0;
         }
 
@@ -1726,7 +1720,7 @@ out:
 	if (ret < 0) {
 		if (transaction_frame)
 			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (setxattr, frame, -1, op_errno);
+		AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
 	}
 
 	return 0;
@@ -1760,7 +1754,7 @@ static int32_t
 pump_truncate (call_frame_t *frame,
                xlator_t *this,
                loc_t *loc,
-               off_t offset)
+               off_t offset, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1770,11 +1764,11 @@ pump_truncate (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->truncate,
                             loc,
-                            offset);
+                            offset, xdata);
                 return 0;
         }
 
-        afr_truncate (frame, this, loc, offset);
+        afr_truncate (frame, this, loc, offset, xdata);
         return 0;
 }
 
@@ -1783,7 +1777,7 @@ static int32_t
 pump_ftruncate (call_frame_t *frame,
                 xlator_t *this,
                 fd_t *fd,
-                off_t offset)
+                off_t offset, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1793,11 +1787,11 @@ pump_ftruncate (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->ftruncate,
                             fd,
-                            offset);
+                            offset, xdata);
                 return 0;
         }
 
-        afr_ftruncate (frame, this, fd, offset);
+        afr_ftruncate (frame, this, fd, offset, xdata);
         return 0;
 }
 
@@ -1806,7 +1800,7 @@ pump_ftruncate (call_frame_t *frame,
 
 int
 pump_mknod (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, mode_t mode, dev_t rdev, dict_t *parms)
+            loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1814,10 +1808,10 @@ pump_mknod (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_mknod_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->mknod,
-                            loc, mode, rdev, parms);
+                            loc, mode, rdev, umask, xdata);
                 return 0;
         }
-        afr_mknod (frame, this, loc, mode, rdev, parms);
+        afr_mknod (frame, this, loc, mode, rdev, umask, xdata);
         return 0;
 
 }
@@ -1826,7 +1820,7 @@ pump_mknod (call_frame_t *frame, xlator_t *this,
 
 int
 pump_mkdir (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, mode_t mode, dict_t *params)
+            loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1834,10 +1828,10 @@ pump_mkdir (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_mkdir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->mkdir,
-                            loc, mode, params);
+                            loc, mode, umask, xdata);
                 return 0;
         }
-        afr_mkdir (frame, this, loc, mode, params);
+        afr_mkdir (frame, this, loc, mode, umask, xdata);
         return 0;
 
 }
@@ -1846,7 +1840,7 @@ pump_mkdir (call_frame_t *frame, xlator_t *this,
 static int32_t
 pump_unlink (call_frame_t *frame,
              xlator_t *this,
-             loc_t *loc)
+             loc_t *loc, int xflag, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1855,10 +1849,10 @@ pump_unlink (call_frame_t *frame,
                             default_unlink_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->unlink,
-                            loc);
+                            loc, xflag, xdata);
                 return 0;
         }
-        afr_unlink (frame, this, loc);
+        afr_unlink (frame, this, loc, xflag, xdata);
         return 0;
 
 }
@@ -1866,7 +1860,7 @@ pump_unlink (call_frame_t *frame,
 
 static int
 pump_rmdir (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int flags)
+            loc_t *loc, int flags, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 
@@ -1876,11 +1870,11 @@ pump_rmdir (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_rmdir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->rmdir,
-                            loc, flags);
+                            loc, flags, xdata);
                 return 0;
         }
 
-        afr_rmdir (frame, this, loc, flags);
+        afr_rmdir (frame, this, loc, flags, xdata);
         return 0;
 
 }
@@ -1889,7 +1883,7 @@ pump_rmdir (call_frame_t *frame, xlator_t *this,
 
 int
 pump_symlink (call_frame_t *frame, xlator_t *this,
-              const char *linkpath, loc_t *loc, dict_t *params)
+              const char *linkpath, loc_t *loc, mode_t umask, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1897,10 +1891,10 @@ pump_symlink (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_symlink_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->symlink,
-                            linkpath, loc, params);
+                            linkpath, loc, umask, xdata);
                 return 0;
         }
-        afr_symlink (frame, this, linkpath, loc, params);
+        afr_symlink (frame, this, linkpath, loc, umask, xdata);
         return 0;
 
 }
@@ -1910,7 +1904,7 @@ static int32_t
 pump_rename (call_frame_t *frame,
              xlator_t *this,
              loc_t *oldloc,
-             loc_t *newloc)
+             loc_t *newloc, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1919,10 +1913,10 @@ pump_rename (call_frame_t *frame,
                             default_rename_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->rename,
-                            oldloc, newloc);
+                            oldloc, newloc, xdata);
                 return 0;
         }
-        afr_rename (frame, this, oldloc, newloc);
+        afr_rename (frame, this, oldloc, newloc, xdata);
         return 0;
 
 }
@@ -1932,7 +1926,7 @@ static int32_t
 pump_link (call_frame_t *frame,
            xlator_t *this,
            loc_t *oldloc,
-           loc_t *newloc)
+           loc_t *newloc, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1941,10 +1935,10 @@ pump_link (call_frame_t *frame,
                             default_link_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->link,
-                            oldloc, newloc);
+                            oldloc, newloc, xdata);
                 return 0;
         }
-        afr_link (frame, this, oldloc, newloc);
+        afr_link (frame, this, oldloc, newloc, xdata);
         return 0;
 
 }
@@ -1953,7 +1947,7 @@ pump_link (call_frame_t *frame,
 static int32_t
 pump_create (call_frame_t *frame, xlator_t *this,
              loc_t *loc, int32_t flags, mode_t mode,
-             fd_t *fd, dict_t *params)
+             mode_t umask, fd_t *fd, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1961,10 +1955,10 @@ pump_create (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_create_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->create,
-                            loc, flags, mode, fd, params);
+                            loc, flags, mode, umask, fd, xdata);
                 return 0;
         }
-        afr_create (frame, this, loc, flags, mode, fd, params);
+        afr_create (frame, this, loc, flags, mode, umask, fd, xdata);
         return 0;
 
 }
@@ -1974,8 +1968,7 @@ static int32_t
 pump_open (call_frame_t *frame,
            xlator_t *this,
            loc_t *loc,
-           int32_t flags, fd_t *fd,
-           int32_t wbflags)
+           int32_t flags, fd_t *fd, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1984,10 +1977,10 @@ pump_open (call_frame_t *frame,
                             default_open_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->open,
-                            loc, flags, fd, wbflags);
+                            loc, flags, fd, xdata);
                 return 0;
         }
-        afr_open (frame, this, loc, flags, fd, wbflags);
+        afr_open (frame, this, loc, flags, fd, xdata);
         return 0;
 
 }
@@ -2000,7 +1993,7 @@ pump_writev (call_frame_t *frame,
              struct iovec *vector,
              int32_t count,
              off_t off, uint32_t flags,
-             struct iobref *iobref)
+             struct iobref *iobref, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2013,19 +2006,19 @@ pump_writev (call_frame_t *frame,
                             vector,
                             count,
                             off, flags,
-                            iobref);
+                            iobref, xdata);
                 return 0;
         }
-        afr_writev (frame, this, fd, vector, count, off, flags, iobref);
-        return 0;
 
+        afr_writev (frame, this, fd, vector, count, off, flags, iobref, xdata);
+        return 0;
 }
 
 
 static int32_t
 pump_flush (call_frame_t *frame,
             xlator_t *this,
-            fd_t *fd)
+            fd_t *fd, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2034,10 +2027,10 @@ pump_flush (call_frame_t *frame,
                             default_flush_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->flush,
-                            fd);
+                            fd, xdata);
                 return 0;
         }
-        afr_flush (frame, this, fd);
+        afr_flush (frame, this, fd, xdata);
         return 0;
 
 }
@@ -2047,7 +2040,7 @@ static int32_t
 pump_fsync (call_frame_t *frame,
             xlator_t *this,
             fd_t *fd,
-            int32_t flags)
+            int32_t flags, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2057,10 +2050,10 @@ pump_fsync (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->fsync,
                             fd,
-                            flags);
+                            flags, xdata);
                 return 0;
         }
-        afr_fsync (frame, this, fd, flags);
+        afr_fsync (frame, this, fd, flags, xdata);
         return 0;
 
 }
@@ -2069,7 +2062,7 @@ pump_fsync (call_frame_t *frame,
 static int32_t
 pump_opendir (call_frame_t *frame,
               xlator_t *this,
-              loc_t *loc, fd_t *fd)
+              loc_t *loc, fd_t *fd, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2078,10 +2071,10 @@ pump_opendir (call_frame_t *frame,
                             default_opendir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->opendir,
-                            loc, fd);
+                            loc, fd, xdata);
                 return 0;
         }
-        afr_opendir (frame, this, loc, fd);
+        afr_opendir (frame, this, loc, fd, xdata);
         return 0;
 
 }
@@ -2091,7 +2084,7 @@ static int32_t
 pump_fsyncdir (call_frame_t *frame,
                xlator_t *this,
                fd_t *fd,
-               int32_t flags)
+               int32_t flags, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2101,10 +2094,10 @@ pump_fsyncdir (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->fsyncdir,
                             fd,
-                            flags);
+                            flags, xdata);
                 return 0;
         }
-        afr_fsyncdir (frame, this, fd, flags);
+        afr_fsyncdir (frame, this, fd, flags, xdata);
         return 0;
 
 }
@@ -2115,7 +2108,7 @@ pump_xattrop (call_frame_t *frame,
               xlator_t *this,
               loc_t *loc,
               gf_xattrop_flags_t flags,
-              dict_t *dict)
+              dict_t *dict, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2126,10 +2119,10 @@ pump_xattrop (call_frame_t *frame,
                             FIRST_CHILD(this)->fops->xattrop,
                             loc,
                             flags,
-                            dict);
+                            dict, xdata);
                 return 0;
         }
-        afr_xattrop (frame, this, loc, flags, dict);
+        afr_xattrop (frame, this, loc, flags, dict, xdata);
         return 0;
 
 }
@@ -2139,7 +2132,7 @@ pump_fxattrop (call_frame_t *frame,
                xlator_t *this,
                fd_t *fd,
                gf_xattrop_flags_t flags,
-               dict_t *dict)
+               dict_t *dict, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2150,10 +2143,10 @@ pump_fxattrop (call_frame_t *frame,
                             FIRST_CHILD(this)->fops->fxattrop,
                             fd,
                             flags,
-                            dict);
+                            dict, xdata);
                 return 0;
         }
-        afr_fxattrop (frame, this, fd, flags, dict);
+        afr_fxattrop (frame, this, fd, flags, dict, xdata);
         return 0;
 
 }
@@ -2163,7 +2156,7 @@ static int32_t
 pump_removexattr (call_frame_t *frame,
                   xlator_t *this,
                   loc_t *loc,
-                  const char *name)
+                  const char *name, dict_t *xdata)
 {
         afr_private_t *priv     = NULL;
         int            op_errno = -1;
@@ -2181,14 +2174,14 @@ pump_removexattr (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->removexattr,
                             loc,
-                            name);
+                            name, xdata);
                 return 0;
         }
-        afr_removexattr (frame, this, loc, name);
+        afr_removexattr (frame, this, loc, name, xdata);
 
  out:
         if (op_errno)
-                AFR_STACK_UNWIND (removexattr, frame, -1, op_errno);
+                AFR_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
         return 0;
 
 }
@@ -2200,7 +2193,7 @@ pump_readdir (call_frame_t *frame,
               xlator_t *this,
               fd_t *fd,
               size_t size,
-              off_t off)
+              off_t off, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2209,10 +2202,10 @@ pump_readdir (call_frame_t *frame,
                             default_readdir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->readdir,
-                            fd, size, off);
+                            fd, size, off, xdata);
                 return 0;
         }
-        afr_readdir (frame, this, fd, size, off);
+        afr_readdir (frame, this, fd, size, off, xdata);
         return 0;
 
 }
@@ -2263,13 +2256,24 @@ pump_release (xlator_t *this,
 
 }
 
+static int32_t
+pump_forget (xlator_t *this, inode_t *inode)
+{
+        afr_private_t  *priv  = NULL;
+
+        priv = this->private;
+        if (priv->use_afr_in_pump)
+                afr_forget (this, inode);
+
+        return 0;
+}
 
 static int32_t
 pump_setattr (call_frame_t *frame,
               xlator_t *this,
               loc_t *loc,
               struct iatt *stbuf,
-              int32_t valid)
+              int32_t valid, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2278,10 +2282,10 @@ pump_setattr (call_frame_t *frame,
                             default_setattr_cbk,
                             FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->setattr,
-                            loc, stbuf, valid);
+                            loc, stbuf, valid, xdata);
                 return 0;
         }
-        afr_setattr (frame, this, loc, stbuf, valid);
+        afr_setattr (frame, this, loc, stbuf, valid, xdata);
         return 0;
 
 }
@@ -2292,7 +2296,7 @@ pump_fsetattr (call_frame_t *frame,
                xlator_t *this,
                fd_t *fd,
                struct iatt *stbuf,
-               int32_t valid)
+               int32_t valid, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2301,10 +2305,10 @@ pump_fsetattr (call_frame_t *frame,
                             default_fsetattr_cbk,
                             FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->fsetattr,
-                            fd, stbuf, valid);
+                            fd, stbuf, valid, xdata);
                 return 0;
         }
-        afr_fsetattr (frame, this, fd, stbuf, valid);
+        afr_fsetattr (frame, this, fd, stbuf, valid, xdata);
         return 0;
 
 }
@@ -2433,14 +2437,13 @@ init (xlator_t *this)
 	priv->metadata_self_heal = 1;
 	priv->entry_self_heal    = 1;
 
-        priv->data_self_heal_algorithm = "";
-
         priv->data_self_heal_window_size = 16;
 
 	priv->data_change_log     = 1;
 	priv->metadata_change_log = 1;
 	priv->entry_change_log    = 1;
         priv->use_afr_in_pump = 1;
+        priv->sh_readdir_size = 65536;
 
 	/* Locking options */
 
@@ -2498,6 +2501,12 @@ init (xlator_t *this)
 		i++;
 	}
 
+        ret = gf_asprintf (&priv->sh_domain, "%s-self-heal", this->name);
+        if (-1 == ret) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
         priv->first_lookup = 1;
         priv->root_inode = NULL;
 
@@ -2529,7 +2538,7 @@ init (xlator_t *this)
                 goto out;
         }
 
-	pump_priv->env = syncenv_new (0);
+	pump_priv->env = this->ctx->env;
         if (!pump_priv->env) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "Could not create new sync-environment");
@@ -2570,9 +2579,6 @@ fini (xlator_t *this)
         if (!pump_priv)
                 goto afr_priv;
 
-        if (pump_priv->env)
-                syncenv_destroy (pump_priv->env);
-
         GF_FREE (pump_priv->resume_path);
         LOCK_DESTROY (&pump_priv->resume_path_lock);
         LOCK_DESTROY (&pump_priv->pump_state_lock);
@@ -2627,6 +2633,7 @@ struct xlator_dumpops dumpops = {
 struct xlator_cbks cbks = {
 	.release     = pump_release,
 	.releasedir  = pump_releasedir,
+        .forget      = pump_forget,
 };
 
 struct volume_options options[] = {
diff --git a/xlators/cluster/afr/src/pump.h b/xlators/cluster/afr/src/pump.h
index 4213a635e..bc4c31a78 100644
--- a/xlators/cluster/afr/src/pump.h
+++ b/xlators/cluster/afr/src/pump.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __PUMP_H__
diff --git a/xlators/cluster/dht/src/Makefile.am b/xlators/cluster/dht/src/Makefile.am
index e35058d65..174bea841 100644
--- a/xlators/cluster/dht/src/Makefile.am
+++ b/xlators/cluster/dht/src/Makefile.am
@@ -4,7 +4,7 @@ xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
 dht_common_source = dht-layout.c dht-helper.c dht-linkfile.c dht-rebalance.c \
 	dht-selfheal.c dht-rename.c dht-hashfn.c dht-diskusage.c \
-	dht-common.c dht-inode-write.c dht-inode-read.c \
+	dht-common.c dht-inode-write.c dht-inode-read.c dht-shared.c \
 	$(top_builddir)/xlators/lib/src/libxlator.c
 
 dht_la_SOURCES = $(dht_common_source) dht.c
@@ -12,22 +12,23 @@ dht_la_SOURCES = $(dht_common_source) dht.c
 nufa_la_SOURCES = $(dht_common_source) nufa.c
 switch_la_SOURCES = $(dht_common_source) switch.c
 
-dht_la_LDFLAGS = -module -avoidversion
+dht_la_LDFLAGS = -module -avoid-version
 dht_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-nufa_la_LDFLAGS = -module -avoidversion
+nufa_la_LDFLAGS = -module -avoid-version
 nufa_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-switch_la_LDFLAGS = -module -avoidversion
+switch_la_LDFLAGS = -module -avoid-version
 switch_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = dht-common.h dht-mem-types.h \
 	$(top_builddir)/xlators/lib/src/libxlator.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS) \
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
 	-I$(top_srcdir)/xlators/lib/src
 
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
 CLEANFILES =
 
 uninstall-local:
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index 059246d97..8f61339e6 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2009-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -31,17 +22,18 @@
 #include "dht-common.h"
 #include "defaults.h"
 #include "byte-order.h"
+#include "glusterfs-acl.h"
 
 #include <sys/time.h>
 #include <libgen.h>
 
-void
+int
 dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
 {
         dict_t  *dst  = NULL;
         int64_t *ptr  = 0, *size = NULL;
         int32_t  ret  = -1;
-        data_pair_t  *data_pair = NULL;
+        data_t  *dict_data = NULL;
 
         dst = data;
 
@@ -53,32 +45,37 @@ dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
                         if (size == NULL) {
                                 gf_log ("dht", GF_LOG_WARNING,
                                         "memory allocation failed");
-                                return;
+                                return -1;
                         }
                         ret = dict_set_bin (dst, key, size, sizeof (int64_t));
                         if (ret < 0) {
                                 gf_log ("dht", GF_LOG_WARNING,
                                         "dht aggregate dict set failed");
                                 GF_FREE (size);
-                                return;
+                                return -1;
                         }
                 }
 
                 ptr = data_to_bin (value);
                 if (ptr == NULL) {
                         gf_log ("dht", GF_LOG_WARNING, "data to bin failed");
-                        return;
+                        return -1;
                 }
 
                 *size = hton64 (ntoh64 (*size) + ntoh64 (*ptr));
+
+        } else if (fnmatch (GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) {
+                ret = gf_get_min_stime (THIS, dst, key, value);
+                if (ret < 0)
+                        return ret;
         } else {
                 /* compare user xattrs only */
                 if (!strncmp (key, "user.", strlen ("user."))) {
-                        ret = dict_lookup (dst, key, &data_pair); 
-                        if (!ret && data_pair && value) {
-                                ret = is_data_equal (data_pair->value, value);
+                        ret = dict_lookup (dst, key, &dict_data);
+                        if (!ret && dict_data && value) {
+                                ret = is_data_equal (dict_data, value);
                                 if (!ret)
-                                        gf_log ("dht", GF_LOG_WARNING,
+                                        gf_log ("dht", GF_LOG_DEBUG,
                                                 "xattr mismatch for %s", key);
                         }
                 }
@@ -87,7 +84,7 @@ dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
                         gf_log ("dht", GF_LOG_WARNING, "xattr dict set failed");
         }
 
-        return;
+        return 0;
 }
 
 
@@ -114,7 +111,7 @@ out:
 int
 dht_lookup_selfheal_cbk (call_frame_t *frame, void *cookie,
                          xlator_t *this,
-                         int op_ret, int op_errno)
+                         int op_ret, int op_errno, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         dht_layout_t *layout = NULL;
@@ -134,7 +131,10 @@ dht_lookup_selfheal_cbk (call_frame_t *frame, void *cookie,
                 ret = dht_layout_set (this, local->inode, layout);
         }
 
-        WIPE (&local->postparent);
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           &local->postparent, 1);
+        }
 
         DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
 
@@ -154,9 +154,11 @@ dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
         int              op_errno = 0;
         int              ret = -1;
         dht_layout_t    *layout = NULL;
+        dht_conf_t      *conf = NULL;
 
         local = discover_frame->local;
         layout = local->layout;
+        conf = this->private;
 
         LOCK(&discover_frame->lock);
         {
@@ -190,16 +192,23 @@ dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
                 }
         } else {
                 ret = dht_layout_normalize (this, &local->loc, layout);
-
-                if (ret != 0) {
+                if ((ret < 0) || ((ret > 0) && (local->op_ret != 0))) {
+                        /* either the layout is incorrect or the directory is
+                         * not found even in one subvolume.
+                         */
                         gf_log (this->name, GF_LOG_DEBUG,
-                                "normalizing failed on %s",
-                                local->loc.path);
-                        op_errno = EINVAL;
-                        goto out;
+                                "normalizing failed on %s "
+                                "(overlaps/holes present: %s, "
+                                "ENOENT errors: %d)", local->loc.path,
+                                (ret < 0) ? "yes" : "no", (ret > 0) ? ret : 0);
+                        if ((ret > 0) && (ret == conf->subvolume_cnt)) {
+                                op_errno = ESTALE;
+                                goto out;
+                        }
                 }
 
-                dht_layout_set (this, local->inode, layout);
+                if (local->inode)
+                        dht_layout_set (this, local->inode, layout);
         }
 
         DHT_STACK_UNWIND (lookup, main_frame, local->op_ret, local->op_errno,
@@ -228,6 +237,7 @@ dht_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         int           is_dir                  = 0;
         int           is_linkfile             = 0;
         int           attempt_unwind          = 0;
+        dht_conf_t   *conf                    = 0;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, out);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -237,6 +247,7 @@ dht_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local = frame->local;
         prev  = cookie;
+        conf = this->private;
 
         layout = local->layout;
 
@@ -271,7 +282,8 @@ dht_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         goto unlock;
                 }
 
-                is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+                is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                                 conf->link_xattr_name);
                 is_dir = check_is_dir (inode, stbuf, xattr);
 
                 if (is_dir) {
@@ -330,23 +342,20 @@ dht_discover (call_frame_t *frame, xlator_t *this, loc_t *loc)
         int          i = 0;
         call_frame_t *discover_frame = NULL;
 
-
         conf = this->private;
         local = frame->local;
 
-        ret = dict_set_uint32 (local->xattr_req,
-                               "trusted.glusterfs.dht", 4 * 4);
+        ret = dict_set_uint32 (local->xattr_req, conf->xattr_name, 4 * 4);
         if (ret)
                 gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set 'trusted.glusterfs.dht' key",
-                        loc->path);
+                        "%s: failed to set '%s' key",
+                        loc->path, conf->xattr_name);
 
-        ret = dict_set_uint32 (local->xattr_req,
-                               "trusted.glusterfs.dht.linkto", 256);
+        ret = dict_set_uint32 (local->xattr_req, conf->link_xattr_name, 256);
         if (ret)
                 gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set 'trusted.glusterfs.dht.linkto' key",
-                        loc->path);
+                        "%s: failed to set '%s' key",
+                        loc->path, conf->link_xattr_name);
 
         call_cnt        = conf->subvolume_cnt;
         local->call_cnt = call_cnt;
@@ -380,7 +389,8 @@ dht_discover (call_frame_t *frame, xlator_t *this, loc_t *loc)
         return 0;
 
 err:
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL);
 
         return 0;
 }
@@ -431,7 +441,7 @@ dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                         op_ret, op_errno, xattr);
 
                 if (op_ret == -1) {
-                        local->op_errno = ENOENT;
+                        local->op_errno = op_errno;
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "lookup of %s on %s returned error (%s)",
                                 local->loc.path, prev->this->name,
@@ -490,6 +500,11 @@ unlock:
                         dht_layout_set (this, local->inode, layout);
                 }
 
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
+
                 DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
                                   local->inode, &local->stbuf, local->xattr,
@@ -521,6 +536,8 @@ dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         int           ret  = -1;
         int           is_dir = 0;
         int           is_linkfile = 0;
+        call_frame_t *copy          = NULL;
+        dht_local_t  *copy_local    = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
         GF_VALIDATE_OR_GOTO ("dht", this, err);
@@ -579,7 +596,8 @@ dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 layout = local->layout;
 
                 is_dir = check_is_dir (inode, stbuf, xattr);
-                is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+                is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                                 conf->link_xattr_name);
 
                 if (is_linkfile) {
                         gf_log (this->name, GF_LOG_INFO,
@@ -591,6 +609,23 @@ dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 }
 
                 if (is_dir) {
+                        ret = dht_dir_has_layout (xattr, conf->xattr_name);
+                        if (ret >= 0) {
+                                if (is_greater_time(local->stbuf.ia_ctime,
+                                                    local->stbuf.ia_ctime_nsec,
+                                                    stbuf->ia_ctime,
+                                                    stbuf->ia_ctime_nsec)) {
+                                        local->prebuf.ia_gid = stbuf->ia_gid;
+                                        local->prebuf.ia_uid = stbuf->ia_uid;
+                                }
+                        }
+                        if (local->stbuf.ia_type != IA_INVAL)
+                        {
+                                if ((local->stbuf.ia_gid != stbuf->ia_gid) ||
+                                    (local->stbuf.ia_uid != stbuf->ia_uid)) {
+                                        local->need_selfheal = 1;
+                                }
+                        }
                         ret = dht_layout_dir_mismatch (this, layout,
                                                        prev->this, &local->loc,
                                                        xattr);
@@ -629,7 +664,28 @@ out:
                     && (conf && conf->unhashed_sticky_bit)) {
                         local->stbuf.ia_prot.sticky = 1;
                 }
-                if (local->layout_mismatch) {
+                if (local->need_selfheal) {
+                        local->need_selfheal = 0;
+                        uuid_copy (local->gfid, local->stbuf.ia_gfid);
+                        local->stbuf.ia_gid = local->prebuf.ia_gid;
+                        local->stbuf.ia_uid = local->prebuf.ia_uid;
+                        copy = create_frame (this, this->ctx->pool);
+                        if (copy) {
+                                copy_local = dht_local_init (copy, &local->loc,
+                                                             NULL, 0);
+                                if (!copy_local)
+                                        goto cont;
+                                copy_local->stbuf = local->stbuf;
+                                copy->local = copy_local;
+                                FRAME_SU_DO (copy, dht_local_t);
+                                ret = synctask_new (this->ctx->env,
+                                                    dht_dir_attr_heal,
+                                                    dht_dir_attr_heal_done,
+                                                    copy, copy);
+                        }
+                }
+cont:
+		if (local->layout_mismatch) {
                         /* Found layout mismatch in the directory, need to
                            fix this in the inode context */
                         dht_layout_unref (this, local->layout);
@@ -655,7 +711,10 @@ out:
                         local->op_errno = ESTALE;
                 }
 
-                WIPE (&local->postparent);
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
 
                 DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
@@ -673,7 +732,8 @@ dht_lookup_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                                 xlator_t *this,
                                 int32_t op_ret, int32_t op_errno,
                                 inode_t *inode, struct iatt *stbuf,
-                                struct iatt *preparent, struct iatt *postparent)
+                                struct iatt *preparent, struct iatt *postparent,
+                                dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         xlator_t     *cached_subvol = NULL;
@@ -706,8 +766,14 @@ dht_lookup_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                 local->stbuf.ia_prot.sticky = 1;
         }
 
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
+
 unwind:
-        WIPE (&local->postparent);
+        if (local->linked == _gf_true)
+                dht_linkfile_attr_heal (frame, this);
 
         DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
         DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
@@ -780,7 +846,10 @@ dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
                                                   "<nil>"));
                 }
 
-                WIPE (&local->postparent);
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
 
                 DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (lookup, frame, local->op_ret,
@@ -810,7 +879,10 @@ dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
                         local->op_errno = EINVAL;
                 }
 
-                WIPE (&local->postparent);
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
 
                 DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (lookup, frame, local->op_ret,
@@ -826,7 +898,7 @@ dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
                 hashed_subvol->name);
 
         ret = dht_linkfile_create (frame,
-                                   dht_lookup_linkfile_create_cbk,
+                                   dht_lookup_linkfile_create_cbk, this,
                                    cached_subvol, hashed_subvol, &local->loc);
 
         return ret;
@@ -836,7 +908,8 @@ dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
 int
 dht_lookup_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int op_ret, int op_errno,
-                       struct iatt *preparent, struct iatt *postparent)
+                       struct iatt *preparent, struct iatt *postparent,
+                       dict_t *xdata)
 {
         int  this_call_cnt = 0;
 
@@ -863,8 +936,9 @@ dht_lookup_everywhere_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         xlator_t     *subvol        = NULL;
         loc_t        *loc           = NULL;
         xlator_t     *link_subvol   = NULL;
-        int           ret = -1;
-        int32_t       fd_count = 0;
+        int           ret           = -1;
+        int32_t       fd_count      = 0;
+        dht_conf_t   *conf          = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, out);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -874,6 +948,7 @@ dht_lookup_everywhere_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local  = frame->local;
         loc    = &local->loc;
+        conf   = this->private;
 
         prev   = cookie;
         subvol = prev->this;
@@ -895,7 +970,8 @@ dht_lookup_everywhere_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                 loc->path, prev->this->name);
                 }
 
-                is_linkfile = check_is_linkfile (inode, buf, xattr);
+                is_linkfile = check_is_linkfile (inode, buf, xattr,
+                                                 conf->link_xattr_name);
                 is_dir = check_is_dir (inode, buf, xattr);
 
                 if (is_linkfile) {
@@ -955,7 +1031,7 @@ unlock:
                                 "deleting stale linkfile %s on %s",
                                 loc->path, subvol->name);
                         STACK_WIND (frame, dht_lookup_unlink_cbk,
-                                    subvol, subvol->fops->unlink, loc);
+                                    subvol, subvol->fops->unlink, loc, 0, NULL);
                         return 0;
                 }
         }
@@ -1037,7 +1113,16 @@ dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
                 gf_log (this->name, GF_LOG_INFO,
                         "lookup of %s on %s (following linkfile) failed (%s)",
                         local->loc.path, subvol->name, strerror (op_errno));
-                goto err;
+
+                /* If cached subvol returned ENOTCONN, do not do
+                lookup_everywhere. We need to make sure linkfile does not get
+                removed, which can take away the namespace, and subvol is
+                anyways down. */
+
+                if (op_errno != ENOTCONN)
+                        goto err;
+                else
+                        goto unwind;
         }
 
         if (check_is_dir (inode, stbuf, xattr)) {
@@ -1047,7 +1132,7 @@ dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
                 goto err;
         }
 
-        if (check_is_linkfile (inode, stbuf, xattr)) {
+        if (check_is_linkfile (inode, stbuf, xattr, conf->link_xattr_name)) {
                 gf_log (this->name, GF_LOG_INFO,
                         "lookup of %s on %s (following linkfile) reached link",
                         local->loc.path, subvol->name);
@@ -1075,9 +1160,12 @@ dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
                 op_errno = EINVAL;
         }
 
-unwind:
-        WIPE (postparent);
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
 
+unwind:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
                           postparent);
@@ -1159,7 +1247,6 @@ dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         loc_t        *loc           = NULL;
         call_frame_t *prev          = NULL;
         int           ret           = 0;
-        uint64_t      tmp_layout    = 0;
         dht_layout_t *parent_layout = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
@@ -1190,8 +1277,10 @@ dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 }
                 if ((conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_AUTO) &&
                     (loc->parent)) {
-                        ret = inode_ctx_get (loc->parent, this, &tmp_layout);
-                        parent_layout = (dht_layout_t *)(long)tmp_layout;
+                        ret = dht_inode_ctx_layout_get (loc->parent, this,
+                                                        &parent_layout);
+                        if (ret || !parent_layout)
+                                goto out;
                         if (parent_layout->search_unhashed) {
                                 local->op_errno = ENOENT;
                                 dht_lookup_everywhere (frame, this, loc);
@@ -1220,7 +1309,8 @@ dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                         conf->link_xattr_name);
 
         if (!is_linkfile) {
                 /* non-directory and not a linkfile */
@@ -1260,7 +1350,10 @@ out:
          * from each of the subvolume. See dht_iatt_merge for reference.
          */
 
-        WIPE (postparent);
+        if (!op_ret && local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
 
         DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
@@ -1269,6 +1362,39 @@ err:
         return 0;
 }
 
+/* For directories, check if acl xattrs have been requested (by the acl xlator),
+ * if not, request for them. These xattrs are needed for dht dir self-heal to
+ * perform proper self-healing of dirs
+ */
+void
+dht_check_and_set_acl_xattr_req (inode_t *inode, dict_t *xattr_req)
+{
+        int     ret = 0;
+
+        GF_ASSERT (inode);
+        GF_ASSERT (xattr_req);
+
+        if (inode->ia_type != IA_IFDIR)
+                return;
+
+        if (!dict_get (xattr_req, POSIX_ACL_ACCESS_XATTR)) {
+                ret = dict_set_int8 (xattr_req, POSIX_ACL_ACCESS_XATTR, 0);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "failed to set key %s",
+                                POSIX_ACL_ACCESS_XATTR);
+        }
+
+        if (!dict_get (xattr_req, POSIX_ACL_DEFAULT_XATTR)) {
+                ret = dict_set_int8 (xattr_req, POSIX_ACL_DEFAULT_XATTR, 0);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "failed to set key %s",
+                                POSIX_ACL_DEFAULT_XATTR);
+        }
+
+        return;
+}
 
 int
 dht_lookup (call_frame_t *frame, xlator_t *this,
@@ -1289,7 +1415,6 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
 
         conf = this->private;
         if (!conf)
@@ -1333,16 +1458,8 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
                 return 0;
         }
 
-        if (!hashed_subvol) {
+        if (!hashed_subvol)
                 hashed_subvol = dht_subvol_get_hashed (this, loc);
-                if (!hashed_subvol) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to get hashed subvol for %s",
-                                loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
-        }
         local->hashed_subvol = hashed_subvol;
 
         if (is_revalidate (loc)) {
@@ -1372,7 +1489,7 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
                  *       revalidates directly go to the cached-subvolume.
                  */
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
+                                       conf->xattr_name, 4 * 4);
 
                 if (IA_ISDIR (local->inode->ia_type)) {
                         local->call_cnt = call_cnt = conf->subvolume_cnt;
@@ -1392,6 +1509,9 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
                 ret = dict_set_uint32 (local->xattr_req,
                                        GLUSTERFS_OPEN_FD_COUNT, 4);
 
+                /* need it for dir self-heal */
+                dht_check_and_set_acl_xattr_req (loc->inode, local->xattr_req);
+
 		for (i = 0; i < call_cnt; i++) {
 			subvol = layout->list[i].xlator;
 
@@ -1404,16 +1524,19 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
         do_fresh_lookup:
                 /* TODO: remove the hard-coding */
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
+                                       conf->xattr_name, 4 * 4);
 
                 ret = dict_set_uint32 (local->xattr_req,
-                                       DHT_LINKFILE_KEY, 256);
+                                       conf->link_xattr_name, 256);
 
                 /* need it for self-healing linkfiles which is
                    'in-migration' state */
                 ret = dict_set_uint32 (local->xattr_req,
                                        GLUSTERFS_OPEN_FD_COUNT, 4);
 
+                /* need it for dir self-heal */
+                dht_check_and_set_acl_xattr_req (loc->inode, local->xattr_req);
+
                 if (!hashed_subvol) {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "no subvolume in layout for path=%s, "
@@ -1447,7 +1570,8 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL);
         return 0;
 }
 
@@ -1455,7 +1579,7 @@ err:
 int
 dht_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int op_ret, int op_errno, struct iatt *preparent,
-                struct iatt *postparent)
+                struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev  = NULL;
@@ -1479,14 +1603,18 @@ dht_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 local->postparent = *postparent;
                 local->preparent = *preparent;
 
-                WIPE (&local->postparent);
-                WIPE (&local->preparent);
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->preparent, 0);
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
         }
 unlock:
         UNLOCK (&frame->lock);
 
         DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent);
+                          &local->preparent, &local->postparent, NULL);
 
         return 0;
 }
@@ -1495,7 +1623,7 @@ unlock:
 int
 dht_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int op_ret, int op_errno, struct iatt *preparent,
-                         struct iatt *postparent)
+                         struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
@@ -1507,7 +1635,8 @@ dht_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         LOCK (&frame->lock);
         {
-                if (op_ret == -1) {
+                if ((op_ret == -1) && !((op_errno == ENOENT) ||
+                                        (op_errno == ENOTCONN))) {
                         local->op_errno = op_errno;
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "subvolume %s returned -1 (%s)",
@@ -1520,7 +1649,7 @@ dht_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 unlock:
         UNLOCK (&frame->lock);
 
-        if (op_ret == -1)
+        if (local->op_ret == -1)
                 goto err;
 
         cached_subvol = dht_subvol_get_cached (this, local->loc.inode);
@@ -1534,53 +1663,19 @@ unlock:
 
         STACK_WIND (frame, dht_unlink_cbk,
                     cached_subvol, cached_subvol->fops->unlink,
-                    &local->loc);
+                    &local->loc, local->flags, NULL);
 
         return 0;
 
 err:
         DHT_STACK_UNWIND (unlink, frame, -1, local->op_errno,
-                          NULL, NULL);
-        return 0;
-}
-
-static int
-dht_ufo_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno)
-{
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
-
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_ret = -1;
-                        local->op_errno = op_errno;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "subvolume %s returned -1 (%s)",
-                                prev->this->name, strerror (op_errno));
-                        goto unlock;
-                }
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret, local->op_errno);
-        }
-
+                          NULL, NULL, NULL);
         return 0;
 }
 
-
 int
 dht_err_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int op_ret, int op_errno)
+             int op_ret, int op_errno, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -1606,7 +1701,8 @@ unlock:
 
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret, local->op_errno);
+                DHT_STACK_UNWIND (setxattr, frame, local->op_ret,
+                                  local->op_errno, NULL);
         }
 
         return 0;
@@ -1649,125 +1745,229 @@ dht_fill_pathinfo_xattr (xlator_t *this, dht_local_t *local,
 }
 
 int
-dht_vgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, dict_t *xattr)
+dht_vgetxattr_alloc_and_fill (dht_local_t *local, dict_t *xattr, xlator_t *this,
+                              int op_errno)
 {
-        dht_local_t *local         = NULL;
-        int          ret           = 0;
-        int          flag          = 0;
-        int          this_call_cnt = 0;
-        char        *value_got     = NULL;
-        char  layout_buf[8192]     = {0,};
-        char        *xattr_buf     = NULL;
-        dict_t      *dict          = NULL;
-        int32_t      alloc_len     = 0;
-        int32_t      plen          = 0;
+        int      ret       = -1;
+        char    *value     = NULL;
+        int32_t  plen      = 0;
 
-        local = frame->local;
+        ret = dict_get_str (xattr, local->xsel, &value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Subvolume %s returned -1 (%s)", this->name,
+                        strerror (op_errno));
+                local->op_ret = -1;
+                local->op_errno = op_errno;
+                goto out;
+        }
 
-        if (op_ret != -1) {
-                ret = dict_get_str (xattr, local->xsel, &value_got);
-                if (!ret) {
-                        alloc_len = strlen (value_got);
+        local->alloc_len += strlen(value);
 
-                        /**
-                         * allocate the buffer:- we allocate 10 bytes extra in
-                         * case we need to append ' Link: ' in the buffer for
-                         * another STACK_WIND
-                         */
+        if (!local->xattr_val) {
+                local->alloc_len += (strlen (DHT_PATHINFO_HEADER) + 10);
+                local->xattr_val = GF_CALLOC (local->alloc_len, sizeof (char),
+                                              gf_common_mt_char);
+                if (!local->xattr_val) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        if (local->xattr_val) {
+                plen = strlen (local->xattr_val);
+                if (plen) {
+                        /* extra byte(s) for \0 to be safe */
+                        local->alloc_len += (plen + 2);
+                        local->xattr_val = GF_REALLOC (local->xattr_val,
+                                                       local->alloc_len);
                         if (!local->xattr_val) {
-                                alloc_len += (strlen (DHT_PATHINFO_HEADER) + 10);
-                                local->xattr_val =
-                                        GF_CALLOC (alloc_len,
-                                                   sizeof (char),
-                                                   gf_common_mt_char);
+                                ret = -1;
+                                goto out;
                         }
+                }
 
-                        if (local->xattr_val) {
-                                plen = strlen (local->xattr_val);
-                                if (plen) {
-                                        /* extra byte(s) for \0 to be safe */
-                                        alloc_len += (plen + 2);
-                                        local->xattr_val =
-                                                GF_REALLOC (local->xattr_val,
-                                                            alloc_len);
-                                        if (!local->xattr_val)
-                                                goto out;
-                                }
+                (void) strcat (local->xattr_val, value);
+                (void) strcat (local->xattr_val, " ");
+                local->op_ret = 0;
+        }
 
-                                strcat (local->xattr_val, value_got);
-                        }
-                }
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+int
+dht_vgetxattr_fill_and_set (dht_local_t *local, dict_t **dict, xlator_t *this,
+                            gf_boolean_t flag)
+{
+        int   ret             = -1;
+        char *xattr_buf       = NULL;
+        char layout_buf[8192] = {0,};
+
+        if (flag)
+                fill_layout_info (local->layout, layout_buf);
+
+        *dict = dict_new ();
+        if (!*dict)
+                goto out;
+
+        local->xattr_val[strlen (local->xattr_val) - 1] = '\0';
+
+        /* we would need max this many bytes to create xattr string
+         * extra 40 bytes is just an estimated amount of additional
+         * space required as we include translator name and some
+         * spaces, brackets etc. when forming the pathinfo string.
+         *
+         * For node-uuid we just don't have all the pretty formatting,
+         * but since this is a generic routine for pathinfo & node-uuid
+         * we dont have conditional space allocation and try to be
+         * generic
+         */
+        local->alloc_len += (2 * strlen (this->name))
+                + strlen (layout_buf)
+                + 40;
+        xattr_buf = GF_CALLOC (local->alloc_len, sizeof (char),
+                               gf_common_mt_char);
+        if (!xattr_buf)
+                goto out;
+
+        if (XATTR_IS_PATHINFO (local->xsel)) {
+                (void) dht_fill_pathinfo_xattr (this, local, xattr_buf,
+                                                local->alloc_len, flag,
+                                                layout_buf);
+        } else if (XATTR_IS_NODE_UUID (local->xsel)) {
+                (void) snprintf (xattr_buf, local->alloc_len, "%s",
+                                 local->xattr_val);
+        } else {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Unknown local->xsel (%s)", local->xsel);
+                goto out;
         }
 
+        ret = dict_set_dynstr (*dict, local->xsel, xattr_buf);
+        GF_FREE (local->xattr_val);
+
  out:
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                if (local->layout->cnt > 1) {
-                        /* Set it for directory */
-                        fill_layout_info (local->layout, layout_buf);
-                        flag = 1;
-                }
+        return ret;
+}
 
-                dict = dict_new ();
-
-                /* we would need max this many bytes to create xattr string */
-                alloc_len += (2 * strlen (this->name))
-                          + strlen (layout_buf)
-                          + 40;
-                xattr_buf = GF_CALLOC (alloc_len,
-                                       sizeof (char), gf_common_mt_char);
-
-                if (XATTR_IS_PATHINFO (local->xsel)) {
-                        (void) dht_fill_pathinfo_xattr (this, local, xattr_buf,
-                                                        alloc_len, flag,
-                                                        layout_buf);
-                } else if (XATTR_IS_NODE_UUID (local->xsel)) {
-                        (void) snprintf (xattr_buf, alloc_len, "%s",
-                                         local->xattr_val);
-                } else
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Unknown local->xsel (%s)", local->xsel);
+int
+dht_vgetxattr_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+        int          ret           = 0;
+        dht_local_t *local         = NULL;
+        int          this_call_cnt = 0;
+        dict_t      *dict          = NULL;
 
-                ret = dict_set_dynstr (dict, local->xsel, xattr_buf);
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (frame->local, out);
 
-                if (local->xattr_val)
-                        GF_FREE (local->xattr_val);
+        local = frame->local;
 
-                DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+        LOCK (&frame->lock);
+        {
+                this_call_cnt = --local->call_cnt;
+                if (op_ret < 0) {
+                        if (op_errno != ENOTCONN) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "getxattr err (%s) for dir",
+                                        strerror (op_errno));
+				local->op_ret = -1;
+				local->op_errno = op_errno;
+                        }
 
-                if (dict)
-                        dict_unref (dict);
+                        goto unlock;
+                }
 
-                return 0;
+                ret = dht_vgetxattr_alloc_and_fill (local, xattr, this,
+                                                    op_errno);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "alloc or fill failure");
         }
+ unlock:
+        UNLOCK (&frame->lock);
 
-        /**
-         * XXX: We will never reach here as call_cnt will always be 1.
-         * But code is kept untouched as we may need it when hashed_subvol
-         * bug is fixed.
-         */
-        if (local->xattr_val)
-                strcat (local->xattr_val, " Link: ");
-        if (local->hashed_subvol) {
-                /* This will happen if there pending */
-                STACK_WIND (frame, dht_vgetxattr_cbk, local->hashed_subvol,
-                            local->hashed_subvol->fops->getxattr,
-                            &local->loc, local->key);
+        if (!is_last_call (this_call_cnt))
+                goto out;
 
-                return 0;
+        /* -- last call: do patch ups -- */
+
+        if (local->op_ret == -1) {
+                goto unwind;
         }
 
-        gf_log ("this->name", GF_LOG_ERROR, "Unable to find hashed_subvol"
-                " for path %s", local->xattr_val);
+        ret = dht_vgetxattr_fill_and_set (local, &dict, this, _gf_true);
+        if (ret)
+                goto unwind;
+
+        DHT_STACK_UNWIND (getxattr, frame, 0, 0, dict, xdata);
+        goto cleanup;
+
+ unwind:
+        DHT_STACK_UNWIND (getxattr, frame, -1, local->op_errno, NULL, NULL);
+ cleanup:
+        if (dict)
+                dict_unref (dict);
+ out:
+        return 0;
+}
+
+int
+dht_vgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+        dht_local_t  *local         = NULL;
+        int           ret           = 0;
+        dict_t       *dict          = NULL;
+        call_frame_t *prev          = NULL;
+        gf_boolean_t  flag          = _gf_true;
+
+        local = frame->local;
+        prev = cookie;
+
+        if (op_ret < 0) {
+                local->op_ret = -1;
+                local->op_errno = op_errno;
+                gf_log (this->name, GF_LOG_ERROR, "Subvolume %s returned -1 "
+                        "(%s)", prev->this->name, strerror (op_errno));
+                goto unwind;
+        }
+
+        ret = dht_vgetxattr_alloc_and_fill (local, xattr, this,
+                                            op_errno);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "alloc or fill failure");
+                goto unwind;
+        }
+
+        flag = (local->layout->cnt > 1) ? _gf_true : _gf_false;
+
+        ret = dht_vgetxattr_fill_and_set (local, &dict, this, flag);
+        if (ret)
+                goto unwind;
+
+        DHT_STACK_UNWIND (getxattr, frame, 0, 0, dict, xdata);
+        goto cleanup;
+
+ unwind:
+        DHT_STACK_UNWIND (getxattr, frame, -1, local->op_errno,
+                          NULL, NULL);
+ cleanup:
+        if (dict)
+                dict_unref (dict);
 
-        DHT_STACK_UNWIND (getxattr, frame, -1, op_errno, dict);
         return 0;
 }
 
 int
 dht_linkinfo_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, dict_t *xattr)
+                           int op_ret, int op_errno, dict_t *xattr,
+                           dict_t *xdata)
 {
         int   ret   = 0;
         char *value = NULL;
@@ -1782,21 +1982,24 @@ dht_linkinfo_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 }
         }
 
-        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr);
+        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
 
         return 0;
 }
 
 int
 dht_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, dict_t *xattr)
+                  int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
         int             this_call_cnt = 0;
         dht_local_t     *local = NULL;
+        dht_conf_t      *conf = NULL;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (frame->local, out);
+        VALIDATE_OR_GOTO (this->private, out);
 
+        conf = this->private;
         local = frame->local;
 
         this_call_cnt = dht_frame_return (frame);
@@ -1804,8 +2007,8 @@ dht_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (!xattr || (op_ret == -1))
                 goto out;
 
-        if (dict_get (xattr, "trusted.glusterfs.dht")) {
-                dict_del (xattr, "trusted.glusterfs.dht");
+        if (dict_get (xattr, conf->xattr_name)) {
+                dict_del (xattr, conf->xattr_name);
         }
         local->op_ret = 0;
 
@@ -1822,24 +2025,88 @@ dht_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 out:
         if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno, local->xattr);
+                DHT_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
+                                  local->xattr, NULL);
         }
         return 0;
 }
 
 int32_t
 dht_getxattr_unwind (call_frame_t *frame,
-                     int op_ret, int op_errno, dict_t *dict)
+                     int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
+dht_getxattr_get_real_filename_cbk (call_frame_t *frame, void *cookie,
+				    xlator_t *this, int op_ret, int op_errno,
+				    dict_t *xattr, dict_t *xdata)
+{
+        int             this_call_cnt = 0;
+        dht_local_t     *local = NULL;
+
+
+        local = frame->local;
+
+	if (op_ret != -1) {
+		if (local->xattr)
+			dict_unref (local->xattr);
+		local->xattr = dict_ref (xattr);
+
+		if (local->xattr_req)
+			dict_unref (local->xattr_req);
+		local->xattr_req = dict_ref (xdata);
+	}
+
+	this_call_cnt = dht_frame_return (frame);
+	if (is_last_call (this_call_cnt)) {
+		DHT_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
+				  local->xattr, local->xattr_req);
+	}
+
+	return 0;
+}
+
+
+int
+dht_getxattr_get_real_filename (call_frame_t *frame, xlator_t *this,
+				loc_t *loc, const char *key, dict_t *xdata)
+{
+	dht_local_t     *local = NULL;
+	int              i = 0;
+	dht_layout_t    *layout = NULL;
+	int              cnt = 0;
+	xlator_t        *subvol = NULL;
+
+
+	local = frame->local;
+	layout = local->layout;
+
+	cnt = local->call_cnt = layout->cnt;
+
+	local->op_ret = -1;
+	local->op_errno = ENODATA;
+
+	for (i = 0; i < cnt; i++) {
+		subvol = layout->list[i].xlator;
+		STACK_WIND (frame, dht_getxattr_get_real_filename_cbk,
+			    subvol, subvol->fops->getxattr,
+			    loc, key, xdata);
+	}
+
+	return 0;
+}
+
+
+int
 dht_getxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, const char *key)
+              loc_t *loc, const char *key, dict_t *xdata)
+#define DHT_IS_DIR(layout)  (layout->cnt > 1)
 {
+
         xlator_t     *subvol        = NULL;
         xlator_t     *hashed_subvol = NULL;
         xlator_t     *cached_subvol = NULL;
@@ -1855,7 +2122,6 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
         VALIDATE_OR_GOTO (this->private, err);
 
         conf   = this->private;
@@ -1883,17 +2149,50 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-        if (key && ((strcmp (key, GF_XATTR_PATHINFO_KEY) == 0)
-                    || strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0)) {
+	if (key &&
+	    (strncmp (key, GF_XATTR_GET_REAL_FILENAME_KEY,
+		      strlen (GF_XATTR_GET_REAL_FILENAME_KEY)) == 0)
+	    && DHT_IS_DIR(layout)) {
+		dht_getxattr_get_real_filename (frame, this, loc, key, xdata);
+		return 0;
+	}
+
+        /* for file use cached subvolume (obviously!): see if {}
+         * below
+         * for directory:
+         *  wind to all subvolumes and exclude subvolumes which
+         *  return ENOTCONN (in callback)
+         *
+         * NOTE: Don't trust inode here, as that may not be valid
+         *       (until inode_link() happens)
+         */
+        if (key && DHT_IS_DIR(layout) &&
+            ((strcmp (key, GF_XATTR_PATHINFO_KEY) == 0)
+             || (strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0))) {
+                (void) strncpy (local->xsel, key, 256);
+                cnt = local->call_cnt = layout->cnt;
+                for (i = 0; i < cnt; i++) {
+                        subvol = layout->list[i].xlator;
+                        STACK_WIND (frame, dht_vgetxattr_dir_cbk,
+                                    subvol, subvol->fops->getxattr,
+                                    loc, key, NULL);
+                }
+                return 0;
+        }
+
+        /* node-uuid or pathinfo for files */
+        if (key && ((strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0)
+                    || (strcmp (key, GF_XATTR_PATHINFO_KEY) == 0))) {
                 cached_subvol = local->cached_subvol;
                 (void) strncpy (local->xsel, key, 256);
 
                 local->call_cnt = 1;
                 STACK_WIND (frame, dht_vgetxattr_cbk, cached_subvol,
-                            cached_subvol->fops->getxattr, loc, key);
+                            cached_subvol->fops->getxattr, loc, key, NULL);
 
                 return 0;
         }
+
         if (key && (strcmp (key, GF_XATTR_LINKINFO_KEY) == 0)) {
                 hashed_subvol = dht_subvol_get_hashed (this, loc);
                 if (!hashed_subvol) {
@@ -1918,7 +2217,7 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                 if (hashed_subvol) {
                         STACK_WIND (frame, dht_linkinfo_getxattr_cbk, hashed_subvol,
                                     hashed_subvol->fops->getxattr, loc,
-                                    GF_XATTR_PATHINFO_KEY);
+                                    GF_XATTR_PATHINFO_KEY, NULL);
                         return 0;
                 }
                 op_errno = ENODATA;
@@ -1926,13 +2225,13 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
         }
 
         if (key && (!strcmp (GF_XATTR_MARKER_KEY, key))
-            && (-1 == frame->root->pid)) {
-
-                if (loc->inode-> ia_type == IA_IFDIR) {
+            && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+                if (DHT_IS_DIR(layout)) {
                         cnt = layout->cnt;
                 } else {
                         cnt = 1;
                 }
+
                 sub_volumes = alloca ( cnt * sizeof (xlator_t *));
                 for (i = 0; i < cnt; i++)
                         *(sub_volumes + i) = layout->list[i].xlator;
@@ -1940,7 +2239,8 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                 if (cluster_getmarkerattr (frame, this, loc, key,
                                            local, dht_getxattr_unwind,
                                            sub_volumes, cnt,
-                                           MARKER_UUID_TYPE, conf->vol_uuid)) {
+                                           MARKER_UUID_TYPE, marker_uuid_default_gauge,
+                                           conf->vol_uuid)) {
                         op_errno = EINVAL;
                         goto err;
                 }
@@ -1950,8 +2250,8 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
 
         if (key && *conf->vol_uuid) {
                 if ((match_uuid_local (key, conf->vol_uuid) == 0) &&
-                    (-1 == frame->root->pid)) {
-                        if (loc->inode-> ia_type == IA_IFDIR) {
+                    (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+                        if (DHT_IS_DIR(layout)) {
                                 cnt = layout->cnt;
                         } else {
                                 cnt = 1;
@@ -1964,6 +2264,7 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                                                    local, dht_getxattr_unwind,
                                                    sub_volumes, cnt,
                                                    MARKER_XTIME_TYPE,
+                                                   marker_xtime_default_gauge,
                                                    conf->vol_uuid)) {
                                 op_errno = EINVAL;
                                 goto err;
@@ -1973,7 +2274,7 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-        if (loc->inode-> ia_type == IA_IFDIR) {
+        if (DHT_IS_DIR(layout)) {
                 cnt = local->call_cnt = layout->cnt;
         } else {
                 cnt = local->call_cnt  = 1;
@@ -1983,20 +2284,21 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                 subvol = layout->list[i].xlator;
                 STACK_WIND (frame, dht_getxattr_cbk,
                             subvol, subvol->fops->getxattr,
-                            loc, key);
+                            loc, key, NULL);
         }
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
+#undef DHT_IS_DIR
 
 int
 dht_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *key)
+               fd_t *fd, const char *key, dict_t *xdata)
 {
         xlator_t     *subvol        = NULL;
         dht_local_t  *local         = NULL;
@@ -2034,7 +2336,9 @@ dht_fgetxattr (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-        if (fd->inode->ia_type == IA_IFDIR) {
+        if ((fd->inode->ia_type == IA_IFDIR)
+            && (strncmp (key, GF_XATTR_LOCKINFO_KEY,
+                         strlen (GF_XATTR_LOCKINFO_KEY) != 0))) {
                 cnt = local->call_cnt = layout->cnt;
         } else {
                 cnt = local->call_cnt  = 1;
@@ -2044,33 +2348,36 @@ dht_fgetxattr (call_frame_t *frame, xlator_t *this,
                 subvol = layout->list[i].xlator;
                 STACK_WIND (frame, dht_getxattr_cbk,
                             subvol, subvol->fops->fgetxattr,
-                            fd, key);
+                            fd, key, NULL);
         }
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
 
 int
 dht_fsetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, dict_t *xattr, int flags)
+               fd_t *fd, dict_t *xattr, int flags, dict_t *xdata)
 {
         xlator_t     *subvol   = NULL;
         dht_local_t  *local    = NULL;
         int           op_errno = EINVAL;
-        data_pair_t  *trav     = NULL;
+        dht_conf_t   *conf     = NULL;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
         VALIDATE_OR_GOTO (fd->inode, err);
+        VALIDATE_OR_GOTO (this->private, err);
+
+        conf = this->private;
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.dht*", xattr,
-                                   trav, op_errno, err);
+        GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
+                                   op_errno, err);
 
         local = dht_local_init (frame, NULL, fd, GF_FOP_FSETXATTR);
         if (!local) {
@@ -2089,13 +2396,13 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this,
         local->call_cnt = 1;
 
         STACK_WIND (frame, dht_err_cbk, subvol, subvol->fops->fsetxattr,
-                    fd, xattr, flags);
+                    fd, xattr, flags, NULL);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsetxattr, frame, -1, op_errno);
+        DHT_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -2103,16 +2410,18 @@ err:
 
 static int
 dht_common_setxattr_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this, int32_t op_ret, int32_t op_errno)
+                         xlator_t *this, int32_t op_ret, int32_t op_errno,
+                         dict_t *xdata)
 {
-        DHT_STACK_UNWIND (setxattr, frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
 
         return 0;
 }
 
 int
 dht_checking_pathinfo_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, dict_t *xattr)
+                           int op_ret, int op_errno, dict_t *xattr,
+                           dict_t *xdata)
 {
         int           i     = -1;
         int           ret   = -1;
@@ -2144,7 +2453,7 @@ dht_checking_pathinfo_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 out:
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret, ENOTSUP);
+                DHT_STACK_UNWIND (setxattr, frame, local->op_ret, ENOTSUP, NULL);
         }
         return 0;
 
@@ -2152,7 +2461,7 @@ out:
 
 int
 dht_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *xattr, int flags)
+              loc_t *loc, dict_t *xattr, int flags, dict_t *xdata)
 {
         xlator_t     *subvol   = NULL;
         dht_local_t  *local    = NULL;
@@ -2166,19 +2475,17 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
         char          value[4096] = {0,};
         gf_dht_migrate_data_type_t forced_rebalance = GF_DHT_MIGRATE_DATA;
         int           call_cnt = 0;
-        data_pair_t  *trav     = NULL;
-
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.dht*", xattr,
-                                   trav, op_errno, err);
 
         conf   = this->private;
+
+        GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
+                                   op_errno, err);
+
         local = dht_local_init (frame, loc, NULL, GF_FOP_SETXATTR);
         if (!local) {
                 op_errno = ENOMEM;
@@ -2203,25 +2510,6 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
 
         local->call_cnt = call_cnt = layout->cnt;
 
-        /*  This key is sent by Unified File and Object storage
-         *  to test xattr support in backend.
-         */
-        tmp = dict_get (xattr, "user.ufo-test");
-        if (tmp) {
-                if (IA_ISREG (loc->inode->ia_type)) {
-                        op_errno = ENOTSUP;
-                        goto err;
-                }
-                local->op_ret = 0;
-                for (i = 0; i < call_cnt; i++) {
-                        STACK_WIND (frame, dht_ufo_xattr_cbk,
-                                    layout->list[i].xlator,
-                                    layout->list[i].xlator->fops->setxattr,
-                                    loc, xattr, flags);
-                }
-                return 0;
-        }
-
         tmp = dict_get (xattr, "distribute.migrate-data");
         if (tmp) {
                 if (IA_ISDIR (loc->inode->ia_type)) {
@@ -2286,7 +2574,7 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
                         STACK_WIND (frame, dht_checking_pathinfo_cbk,
                                     conf->subvolumes[i],
                                     conf->subvolumes[i]->fops->getxattr,
-                                    loc, GF_XATTR_PATHINFO_KEY);
+                                    loc, GF_XATTR_PATHINFO_KEY, NULL);
                 }
                 return 0;
         }
@@ -2296,9 +2584,13 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
                 gf_log (this->name, GF_LOG_INFO,
                         "fixing the layout of %s", loc->path);
 
-                dht_fix_directory_layout (frame, dht_common_setxattr_cbk,
-                                          layout);
-                return 0;
+                ret = dht_fix_directory_layout (frame, dht_common_setxattr_cbk,
+                                                layout);
+                if (ret) {
+                        op_errno = ENOTCONN;
+                        goto err;
+                }
+                return ret;
         }
 
         tmp = dict_get (xattr, "distribute.directory-spread-count");
@@ -2310,10 +2602,14 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
                              (dir_spread > 0))) {
                         layout->spread_cnt = dir_spread;
 
-                        dht_fix_directory_layout (frame,
-                                                  dht_common_setxattr_cbk,
-                                                  layout);
-                        return 0;
+                        ret = dht_fix_directory_layout (frame,
+                                                        dht_common_setxattr_cbk,
+                                                        layout);
+                        if (ret) {
+                                op_errno = ENOTCONN;
+                                goto err;
+                        }
+                        return ret;
                 }
                 gf_log (this->name, GF_LOG_ERROR,
                         "wrong 'directory-spread-count' value (%s)", value);
@@ -2325,14 +2621,14 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, dht_err_cbk,
                             layout->list[i].xlator,
                             layout->list[i].xlator->fops->setxattr,
-                            loc, xattr, flags);
+                            loc, xattr, flags, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (setxattr, frame, -1, op_errno);
+        DHT_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -2340,7 +2636,7 @@ err:
 
 int
 dht_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno)
+                     int op_ret, int op_errno, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -2366,7 +2662,8 @@ unlock:
 
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (removexattr, frame, local->op_ret, local->op_errno);
+                DHT_STACK_UNWIND (removexattr, frame, local->op_ret,
+                                  local->op_errno, NULL);
         }
 
         return 0;
@@ -2375,25 +2672,27 @@ unlock:
 
 int
 dht_removexattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *key)
+                 loc_t *loc, const char *key, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
         dht_local_t  *local = NULL;
         dht_layout_t *layout = NULL;
         int           call_cnt = 0;
+        dht_conf_t   *conf = NULL;
 
         int i;
 
         VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (this->private, err);
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.dht*",
-                                 key, op_errno, err);
+        conf = this->private;
+
+        GF_IF_NATIVE_XATTR_GOTO (conf->wild_xattr_name, key, op_errno, err);
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
 
         local = dht_local_init (frame, loc, NULL, GF_FOP_REMOVEXATTR);
         if (!local) {
@@ -2424,34 +2723,37 @@ dht_removexattr (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, dht_removexattr_cbk,
                             layout->list[i].xlator,
                             layout->list[i].xlator->fops->removexattr,
-                            loc, key);
+                            loc, key, NULL);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (removexattr, frame, -1, op_errno);
+        DHT_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
 
         return 0;
 }
 
 int
 dht_fremovexattr (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, const char *key)
+                  fd_t *fd, const char *key, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
         dht_local_t  *local = NULL;
         dht_layout_t *layout = NULL;
         int           call_cnt = 0;
+        dht_conf_t   *conf = 0;
 
         int i;
 
         VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (this->private, err);
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.dht*",
-                                 key, op_errno, err);
+        conf = this->private;
+
+        GF_IF_NATIVE_XATTR_GOTO (conf->wild_xattr_name, key, op_errno, err);
 
         VALIDATE_OR_GOTO (frame, err);
 
@@ -2485,14 +2787,14 @@ dht_fremovexattr (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, dht_removexattr_cbk,
                             layout->list[i].xlator,
                             layout->list[i].xlator->fops->fremovexattr,
-                            fd, key);
+                            fd, key, NULL);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fremovexattr, frame, -1, op_errno);
+        DHT_STACK_UNWIND (fremovexattr, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -2500,7 +2802,7 @@ err:
 
 int
 dht_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int op_ret, int op_errno, fd_t *fd)
+            int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -2527,7 +2829,7 @@ unlock:
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt))
                 DHT_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
-                                  local->fd);
+                                  local->fd, NULL);
 
         return 0;
 }
@@ -2557,7 +2859,7 @@ dht_normalize_stats (struct statvfs *buf, unsigned long bsize,
 
 int
 dht_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct statvfs *statvfs)
+                int op_ret, int op_errno, struct statvfs *statvfs, dict_t *xdata)
 {
         dht_local_t *local         = NULL;
         int          this_call_cnt = 0;
@@ -2603,14 +2905,14 @@ unlock:
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt))
                 DHT_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
-                                  &local->statvfs);
+                                  &local->statvfs, xdata);
 
         return 0;
 }
 
 
 int
-dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         dht_local_t  *local  = NULL;
@@ -2622,7 +2924,6 @@ dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
         VALIDATE_OR_GOTO (this->private, err);
 
         conf = this->private;
@@ -2639,7 +2940,8 @@ dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 for (i = 0; i < conf->subvolume_cnt; i++) {
                         STACK_WIND (frame, dht_statfs_cbk,
                                     conf->subvolumes[i],
-                                    conf->subvolumes[i]->fops->statfs, loc);
+                                    conf->subvolumes[i]->fops->statfs, loc,
+                                    xdata);
                 }
                 return 0;
         }
@@ -2655,20 +2957,21 @@ dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
         local->call_cnt = 1;
 
         STACK_WIND (frame, dht_statfs_cbk,
-                    subvol, subvol->fops->statfs, loc);
+                    subvol, subvol->fops->statfs, loc, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (statfs, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
 
 
 int
-dht_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+dht_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+             dict_t *xdata)
 {
         dht_local_t  *local  = NULL;
         dht_conf_t   *conf = NULL;
@@ -2695,14 +2998,14 @@ dht_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
                 STACK_WIND (frame, dht_fd_cbk,
                             conf->subvolumes[i],
                             conf->subvolumes[i]->fops->opendir,
-                            loc, fd);
+                            loc, fd, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (opendir, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (opendir, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -2710,7 +3013,7 @@ err:
 
 int
 dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-                  int op_errno, gf_dirent_t *orig_entries)
+                  int op_errno, gf_dirent_t *orig_entries, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         gf_dirent_t   entries;
@@ -2723,6 +3026,7 @@ dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
         dht_layout_t *layout = 0;
         dht_conf_t   *conf   = NULL;
         xlator_t     *subvol = 0;
+        int           ret    = 0;
 
         INIT_LIST_HEAD (&entries.list);
         prev = cookie;
@@ -2739,10 +3043,13 @@ dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
 
         list_for_each_entry (orig_entry, (&orig_entries->list), list) {
                 next_offset = orig_entry->d_off;
-                if ((check_is_dir (NULL, (&orig_entry->d_stat), NULL) &&
-                     (prev->this != dht_first_up_subvol (this))) ||
-                    check_is_linkfile (NULL, (&orig_entry->d_stat),
-                                       orig_entry->dict)) {
+                if (check_is_dir (NULL, (&orig_entry->d_stat), NULL) &&
+                    (prev->this != local->first_up_subvol)) {
+                        continue;
+                }
+                if (check_is_linkfile (NULL, (&orig_entry->d_stat),
+                                       orig_entry->dict,
+                                       conf->link_xattr_name)) {
                         continue;
                 }
 
@@ -2771,6 +3078,24 @@ dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
                 entry->d_type = orig_entry->d_type;
                 entry->d_len  = orig_entry->d_len;
 
+                if (orig_entry->dict)
+                        entry->dict = dict_ref (orig_entry->dict);
+
+                /* making sure we set the inode ctx right with layout,
+                   currently possible only for non-directories, so for
+                   directories don't set entry inodes */
+                if (!IA_ISDIR(entry->d_stat.ia_type)) {
+                        ret = dht_layout_preset (this, prev->this,
+                                                 orig_entry->inode);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "failed to link the layout in inode");
+                        entry->inode = inode_ref (orig_entry->inode);
+                } else if (orig_entry->inode) {
+                        dht_inode_ctx_time_update (orig_entry->inode, this,
+                                                   &entry->d_stat, 1);
+                }
+
                 list_add_tail (&entry->list, &entries.list);
                 count++;
         }
@@ -2800,6 +3125,19 @@ done:
                         goto unwind;
                 }
 
+		if (conf->readdir_optimize == _gf_true) {
+                        if (next_subvol != local->first_up_subvol) {
+                                ret = dict_set_int32 (local->xattr,
+                                                      GF_READDIR_SKIP_DIRS, 1);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+					         "dict set failed");
+		        } else {
+                                 dict_del (local->xattr,
+                                           GF_READDIR_SKIP_DIRS);
+                        }
+                }
+
                 STACK_WIND (frame, dht_readdirp_cbk,
                             next_subvol, next_subvol->fops->readdirp,
                             local->fd, local->size, next_offset,
@@ -2811,7 +3149,7 @@ unwind:
         if (op_ret < 0)
                 op_ret = 0;
 
-        DHT_STACK_UNWIND (readdirp, frame, op_ret, op_errno, &entries);
+        DHT_STACK_UNWIND (readdirp, frame, op_ret, op_errno, &entries, NULL);
 
         gf_dirent_free (&entries);
 
@@ -2822,7 +3160,8 @@ unwind:
 
 int
 dht_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, gf_dirent_t *orig_entries)
+                 int op_ret, int op_errno, gf_dirent_t *orig_entries,
+                 dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         gf_dirent_t   entries;
@@ -2899,7 +3238,7 @@ done:
 
                 STACK_WIND (frame, dht_readdir_cbk,
                             next_subvol, next_subvol->fops->readdir,
-                            local->fd, local->size, next_offset);
+                            local->fd, local->size, next_offset, NULL);
                 return 0;
         }
 
@@ -2907,7 +3246,7 @@ unwind:
         if (op_ret < 0)
                 op_ret = 0;
 
-        DHT_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries);
+        DHT_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries, NULL);
 
         gf_dirent_free (&entries);
 
@@ -2924,10 +3263,14 @@ dht_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
         xlator_t     *xvol = NULL;
         off_t         xoff = 0;
         int           ret = 0;
+        dht_conf_t   *conf = NULL;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (this->private, err);
+
+        conf = this->private;
 
         local = dht_local_init (frame, NULL, NULL, whichop);
         if (!local) {
@@ -2938,6 +3281,7 @@ dht_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
         local->fd = fd_ref (fd);
         local->size = size;
         local->xattr_req = (dict)? dict_ref (dict) : NULL;
+        local->first_up_subvol = dht_first_up_subvol (this);
 
         dht_deitransform (this, yoff, &xvol, (uint64_t *)&xoff);
 
@@ -2950,26 +3294,38 @@ dht_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 
                 if (local->xattr) {
                         ret = dict_set_uint32 (local->xattr,
-                                               "trusted.glusterfs.dht.linkto",
-                                               256);
+                                               conf->link_xattr_name, 256);
                         if (ret)
                                 gf_log (this->name, GF_LOG_WARNING,
-                                        "failed to set 'glusterfs.dht.linkto'"
-                                        " key");
+                                        "failed to set '%s' key",
+                                        conf->link_xattr_name);
+			if (conf->readdir_optimize == _gf_true) {
+                                if (xvol != local->first_up_subvol) {
+				        ret = dict_set_int32 (local->xattr,
+			                               GF_READDIR_SKIP_DIRS, 1);
+				        if (ret)
+					        gf_log (this->name,
+                                                        GF_LOG_ERROR,
+						        "Dict set failed");
+                                } else {
+                                        dict_del (local->xattr,
+                                                  GF_READDIR_SKIP_DIRS);
+                                }
+			}
                 }
 
                 STACK_WIND (frame, dht_readdirp_cbk, xvol, xvol->fops->readdirp,
                             fd, size, xoff, local->xattr);
         } else {
                 STACK_WIND (frame, dht_readdir_cbk, xvol, xvol->fops->readdir,
-                            fd, size, xoff);
+                            fd, size, xoff, local->xattr);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readdir, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -2977,7 +3333,7 @@ err:
 
 int
 dht_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t yoff)
+             off_t yoff, dict_t *xdata)
 {
         int          op = GF_FOP_READDIR;
         dht_conf_t  *conf = NULL;
@@ -3014,7 +3370,7 @@ dht_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 
 int
 dht_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno)
+                  int op_ret, int op_errno, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -3034,14 +3390,16 @@ dht_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt))
-                DHT_STACK_UNWIND (fsyncdir, frame, local->op_ret, local->op_errno);
+                DHT_STACK_UNWIND (fsyncdir, frame, local->op_ret,
+                                  local->op_errno, xdata);
 
         return 0;
 }
 
 
 int
-dht_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync)
+dht_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              int datasync, dict_t *xdata)
 {
         dht_local_t  *local  = NULL;
         dht_conf_t   *conf = NULL;
@@ -3068,14 +3426,14 @@ dht_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync)
                 STACK_WIND (frame, dht_fsyncdir_cbk,
                             conf->subvolumes[i],
                             conf->subvolumes[i]->fops->fsyncdir,
-                            fd, datasync);
+                            fd, datasync, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsyncdir, frame, -1, op_errno);
+        DHT_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -3085,9 +3443,9 @@ int
 dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int op_ret, int op_errno,
                  inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-                 struct iatt *postparent)
+                 struct iatt *postparent, dict_t *xdata)
 {
-        call_frame_t *prev = NULL;
+        xlator_t     *prev = NULL;
         int           ret = -1;
         dht_local_t  *local = NULL;
 
@@ -3105,19 +3463,24 @@ dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         prev = cookie;
 
         if (local->loc.parent) {
-                WIPE (preparent);
-                WIPE (postparent);
+
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           preparent, 0);
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
         }
 
-        ret = dht_layout_preset (this, prev->this, inode);
+        ret = dht_layout_preset (this, prev, inode);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "could not set pre-set layout for subvolume %s",
-                        prev->this->name);
+                        prev? prev->name: NULL);
                 op_ret   = -1;
                 op_errno = EINVAL;
                 goto out;
         }
+        if (local->linked == _gf_true)
+                dht_linkfile_attr_heal (frame, this);
 out:
         /*
          * FIXME: ia_size and st_blocks of preparent and postparent do not have
@@ -3126,10 +3489,9 @@ out:
          * corresponding values from each of the subvolume.
          * See dht_iatt_merge for reference.
          */
-
         DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, stbuf, preparent,
-                          postparent);
+        DHT_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, stbuf,
+                          preparent, postparent, xdata);
         return 0;
 }
 
@@ -3138,7 +3500,8 @@ dht_mknod_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                                xlator_t *this,
                                int32_t op_ret, int32_t op_errno,
                                inode_t *inode, struct iatt *stbuf,
-                               struct iatt *preparent, struct iatt *postparent)
+                               struct iatt *preparent, struct iatt *postparent,
+                               dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         xlator_t     *cached_subvol = NULL;
@@ -3147,22 +3510,28 @@ dht_mknod_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                 goto err;
 
         local = frame->local;
+        if (!local || !local->cached_subvol) {
+                op_errno = EINVAL;
+                goto err;
+        }
+
         cached_subvol = local->cached_subvol;
 
-        STACK_WIND (frame, dht_newfile_cbk,
-                    cached_subvol, cached_subvol->fops->mknod,
-                    &local->loc, local->mode, local->rdev,
-                    local->params);
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)cached_subvol,
+                           cached_subvol, cached_subvol->fops->mknod,
+                           &local->loc, local->mode, local->rdev, local->umask,
+                           local->params);
 
         return 0;
 err:
-        DHT_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                          NULL);
         return 0;
 }
 
 int
 dht_mknod (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, dev_t rdev, dict_t *params)
+           loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *params)
 {
         xlator_t    *subvol = NULL;
         int          op_errno = -1;
@@ -3194,11 +3563,13 @@ dht_mknod (call_frame_t *frame, xlator_t *this,
                 gf_log (this->name, GF_LOG_TRACE,
                         "creating %s on %s", loc->path, subvol->name);
 
-                STACK_WIND (frame, dht_newfile_cbk,
-                            subvol, subvol->fops->mknod,
-                            loc, mode, rdev, params);
+                STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol,
+                                   subvol, subvol->fops->mknod, loc, mode,
+                                   rdev, umask, params);
         } else {
-                avail_subvol = dht_free_disk_available_subvol (this, subvol);
+
+                avail_subvol = dht_free_disk_available_subvol (this, subvol,
+                                                               local);
                 if (avail_subvol != subvol) {
                         /* Choose the minimum filled volume, and create the
                            files there */
@@ -3207,17 +3578,18 @@ dht_mknod (call_frame_t *frame, xlator_t *this,
                         local->cached_subvol = avail_subvol;
                         local->mode = mode;
                         local->rdev = rdev;
-
+                        local->umask = umask;
                         dht_linkfile_create (frame,
                                              dht_mknod_linkfile_create_cbk,
-                                             avail_subvol, subvol, loc);
+                                             this, avail_subvol, subvol, loc);
                 } else {
                         gf_log (this->name, GF_LOG_TRACE,
                                 "creating %s on %s", loc->path, subvol->name);
 
-                        STACK_WIND (frame, dht_newfile_cbk,
-                                    subvol, subvol->fops->mknod,
-                                    loc, mode, rdev, params);
+                        STACK_WIND_COOKIE (frame, dht_newfile_cbk,
+                                           (void *)subvol, subvol,
+                                           subvol->fops->mknod, loc, mode,
+                                           rdev, umask, params);
                 }
         }
 
@@ -3226,7 +3598,7 @@ dht_mknod (call_frame_t *frame, xlator_t *this,
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
         DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL);
+                          NULL, NULL, NULL, NULL, NULL);
 
         return 0;
 }
@@ -3234,7 +3606,7 @@ err:
 
 int
 dht_symlink (call_frame_t *frame, xlator_t *this,
-             const char *linkname, loc_t *loc, dict_t *params)
+             const char *linkname, loc_t *loc, mode_t umask, dict_t *params)
 {
         xlator_t    *subvol = NULL;
         int          op_errno = -1;
@@ -3262,23 +3634,24 @@ dht_symlink (call_frame_t *frame, xlator_t *this,
         gf_log (this->name, GF_LOG_TRACE,
                 "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND (frame, dht_newfile_cbk,
-                    subvol, subvol->fops->symlink,
-                    linkname, loc, params);
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
+                           subvol->fops->symlink, linkname, loc, umask,
+                           params);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
         DHT_STACK_UNWIND (link, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL);
+                          NULL, NULL, NULL, NULL, NULL);
 
         return 0;
 }
 
 
 int
-dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+            dict_t *xdata)
 {
         xlator_t    *cached_subvol = NULL;
         xlator_t    *hashed_subvol = NULL;
@@ -3296,7 +3669,7 @@ dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
                         local->loc.path, cached_subvol->name, loc->path);
                 STACK_WIND (frame, dht_unlink_cbk,
                             cached_subvol, cached_subvol->fops->unlink,
-                            &local->loc);
+                            &local->loc, xflag, xdata);
                 goto done;
         }
 
@@ -3324,18 +3697,21 @@ dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 goto err;
         }
 
+        local->flags = xflag;
         if (hashed_subvol != cached_subvol) {
                 STACK_WIND (frame, dht_unlink_linkfile_cbk,
-                            hashed_subvol, hashed_subvol->fops->unlink, loc);
+                            hashed_subvol, hashed_subvol->fops->unlink, loc,
+                            xflag, xdata);
         } else {
                 STACK_WIND (frame, dht_unlink_cbk,
-                            cached_subvol, cached_subvol->fops->unlink, loc);
+                            cached_subvol, cached_subvol->fops->unlink, loc,
+                            xflag, xdata);
         }
 done:
         return 0;
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -3345,13 +3721,16 @@ int
 dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
               int op_ret, int op_errno,
               inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-              struct iatt *postparent)
+              struct iatt *postparent, dict_t *xdata)
 {
         call_frame_t *prev = NULL;
         dht_layout_t *layout = NULL;
+        dht_local_t  *local = NULL;
 
         prev = cookie;
 
+        local = frame->local;
+
         if (op_ret == -1)
                 goto out;
 
@@ -3365,13 +3744,20 @@ dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        WIPE (preparent);
-        WIPE (postparent);
-
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           preparent, 0);
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
+        if (local->linked == _gf_true) {
+                local->stbuf = *stbuf;
+                dht_linkfile_attr_heal (frame, this);
+        }
 out:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf, preparent,
-                          postparent);
+                          postparent, NULL);
 
         return 0;
 }
@@ -3381,7 +3767,8 @@ int
 dht_link_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int op_ret, int op_errno,
                        inode_t *inode, struct iatt *stbuf,
-                       struct iatt *preparent, struct iatt *postparent)
+                       struct iatt *preparent, struct iatt *postparent,
+                       dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         xlator_t     *srcvol = NULL;
@@ -3393,14 +3780,14 @@ dht_link_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         srcvol = local->linkfile.srcvol;
 
         STACK_WIND (frame, dht_link_cbk, srcvol, srcvol->fops->link,
-                    &local->loc, &local->loc2);
+                    &local->loc, &local->loc2, xdata);
 
         return 0;
 
 err:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf, preparent,
-                          postparent);
+                          postparent, NULL);
 
         return 0;
 }
@@ -3408,7 +3795,7 @@ err:
 
 int
 dht_link (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc)
+          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
         xlator_t    *cached_subvol = NULL;
         xlator_t    *hashed_subvol = NULL;
@@ -3453,19 +3840,19 @@ dht_link (call_frame_t *frame, xlator_t *this,
 
         if (hashed_subvol != cached_subvol) {
                 uuid_copy (local->gfid, oldloc->inode->gfid);
-                dht_linkfile_create (frame, dht_link_linkfile_cbk,
+                dht_linkfile_create (frame, dht_link_linkfile_cbk, this,
                                      cached_subvol, hashed_subvol, newloc);
         } else {
                 STACK_WIND (frame, dht_link_cbk,
                             cached_subvol, cached_subvol->fops->link,
-                            oldloc, newloc);
+                            oldloc, newloc, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
         return 0;
 }
@@ -3475,7 +3862,7 @@ int
 dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int op_ret, int op_errno,
                 fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                struct iatt *preparent, struct iatt *postparent)
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         call_frame_t *prev = NULL;
         int           ret = -1;
@@ -3494,8 +3881,11 @@ dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         prev = cookie;
 
         if (local->loc.parent) {
-                WIPE (preparent);
-                WIPE (postparent);
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           preparent, 0);
+
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
         }
 
         ret = dht_layout_preset (this, prev->this, inode);
@@ -3507,11 +3897,14 @@ dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 op_errno = EINVAL;
                 goto out;
         }
-
+        if (local->linked == _gf_true) {
+                local->stbuf = *stbuf;
+                dht_linkfile_attr_heal (frame, this);
+        }
 out:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, stbuf, preparent,
-                          postparent);
+                          postparent, NULL);
         return 0;
 }
 
@@ -3521,7 +3914,8 @@ dht_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                                 xlator_t *this,
                                 int32_t op_ret, int32_t op_errno,
                                 inode_t *inode, struct iatt *stbuf,
-                                struct iatt *preparent, struct iatt *postparent)
+                                struct iatt *preparent, struct iatt *postparent,
+                                dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         xlator_t     *cached_subvol = NULL;
@@ -3535,18 +3929,19 @@ dht_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
         STACK_WIND (frame, dht_create_cbk,
                     cached_subvol, cached_subvol->fops->create,
                     &local->loc, local->flags, local->mode,
-                    local->fd, local->params);
+                    local->umask, local->fd, local->params);
 
         return 0;
 err:
-        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL, NULL, NULL);
         return 0;
 }
 
 int
 dht_create (call_frame_t *frame, xlator_t *this,
             loc_t *loc, int32_t flags, mode_t mode,
-            fd_t *fd, dict_t *params)
+            mode_t umask, fd_t *fd, dict_t *params)
 {
         int          op_errno = -1;
         xlator_t    *subvol = NULL;
@@ -3572,7 +3967,7 @@ dht_create (call_frame_t *frame, xlator_t *this,
                         local->loc.path, subvol->name, loc->path);
                 STACK_WIND (frame, dht_create_cbk,
                             subvol, subvol->fops->create,
-                            &local->loc, flags, mode, fd, params);
+                            &local->loc, flags, mode, umask, fd, params);
                 goto done;
         }
 
@@ -3590,38 +3985,38 @@ dht_create (call_frame_t *frame, xlator_t *this,
                         "creating %s on %s", loc->path, subvol->name);
                 STACK_WIND (frame, dht_create_cbk,
                             subvol, subvol->fops->create,
-                            loc, flags, mode, fd, params);
+                            loc, flags, mode, umask, fd, params);
                 goto done;
         }
         /* Choose the minimum filled volume, and create the
            files there */
-        avail_subvol = dht_free_disk_available_subvol (this, subvol);
+        avail_subvol = dht_free_disk_available_subvol (this, subvol, local);
         if (avail_subvol != subvol) {
                 local->params = dict_ref (params);
                 local->flags = flags;
                 local->mode = mode;
-
+                local->umask = umask;
                 local->cached_subvol = avail_subvol;
                 local->hashed_subvol = subvol;
                 gf_log (this->name, GF_LOG_TRACE,
                         "creating %s on %s (link at %s)", loc->path,
                         avail_subvol->name, subvol->name);
-                dht_linkfile_create (frame,
-                                     dht_create_linkfile_create_cbk,
-                                     avail_subvol, subvol, loc);
+                dht_linkfile_create (frame, dht_create_linkfile_create_cbk,
+                                     this, avail_subvol, subvol, loc);
                 goto done;
         }
         gf_log (this->name, GF_LOG_TRACE,
                 "creating %s on %s", loc->path, subvol->name);
         STACK_WIND (frame, dht_create_cbk,
                     subvol, subvol->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, params);
 done:
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL, NULL, NULL);
 
         return 0;
 }
@@ -3630,7 +4025,7 @@ err:
 int
 dht_mkdir_selfheal_cbk (call_frame_t *frame, void *cookie,
                         xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         dht_local_t   *local = NULL;
         dht_layout_t  *layout = NULL;
@@ -3641,14 +4036,17 @@ dht_mkdir_selfheal_cbk (call_frame_t *frame, void *cookie,
         if (op_ret == 0) {
                 dht_layout_set (this, local->inode, layout);
                 if (local->loc.parent) {
-                        WIPE (&local->preparent);
-                        WIPE (&local->postparent);
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->preparent, 0);
+
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
                 }
         }
 
         DHT_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
                           local->inode, &local->stbuf, &local->preparent,
-                          &local->postparent);
+                          &local->postparent, NULL);
 
         return 0;
 }
@@ -3656,7 +4054,7 @@ dht_mkdir_selfheal_cbk (call_frame_t *frame, void *cookie,
 int
 dht_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
-               struct iatt *preparent, struct iatt *postparent)
+               struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -3677,6 +4075,15 @@ dht_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         ret = dht_layout_merge (this, layout, prev->this,
                                                 -1, ENOSPC, NULL);
                 } else {
+			if (op_ret == -1 && op_errno == EEXIST)
+				/* Very likely just a race between mkdir and
+				   self-heal (from lookup of a concurrent mkdir
+				   attempt).
+				   Ignore error for now. layout setting will
+				   anyways fail if this was a different (old)
+				   pre-existing different directory.
+				*/
+				op_ret = 0;
                         ret = dht_layout_merge (this, layout, prev->this,
                                                 op_ret, op_errno, NULL);
                 }
@@ -3709,7 +4116,8 @@ int
 dht_mkdir_hashed_cbk (call_frame_t *frame, void *cookie,
                       xlator_t *this, int op_ret, int op_errno,
                       inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent)
+                      struct iatt *preparent, struct iatt *postparent,
+                      dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           ret = -1;
@@ -3766,19 +4174,20 @@ dht_mkdir_hashed_cbk (call_frame_t *frame, void *cookie,
                         continue;
                 STACK_WIND (frame, dht_mkdir_cbk,
                             conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->mkdir,
-                            &local->loc, local->mode, local->params);
+                            conf->subvolumes[i]->fops->mkdir, &local->loc,
+                            local->mode, local->umask, local->params);
         }
         return 0;
 err:
-        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL, NULL);
         return 0;
 }
 
 
  int
 dht_mkdir (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, dict_t *params)
+           loc_t *loc, mode_t mode, mode_t umask, dict_t *params)
 {
         dht_local_t  *local  = NULL;
         dht_conf_t   *conf = NULL;
@@ -3814,6 +4223,7 @@ dht_mkdir (call_frame_t *frame, xlator_t *this,
 
         local->hashed_subvol = hashed_subvol;
         local->mode = mode;
+        local->umask = umask;
         local->params = dict_ref (params);
         local->inode  = inode_ref (loc->inode);
 
@@ -3826,13 +4236,14 @@ dht_mkdir (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, dht_mkdir_hashed_cbk,
                     hashed_subvol,
                     hashed_subvol->fops->mkdir,
-                    loc, mode, params);
+                    loc, mode, umask, params);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL, NULL);
 
         return 0;
 }
@@ -3840,14 +4251,87 @@ err:
 
 int
 dht_rmdir_selfheal_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno)
+                        int op_ret, int op_errno, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
 
         local = frame->local;
 
         DHT_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent);
+                          &local->preparent, &local->postparent, NULL);
+
+        return 0;
+}
+
+
+int
+dht_rmdir_hashed_subvol_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int op_ret, int op_errno, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
+        call_frame_t *prev = NULL;
+
+        local = frame->local;
+        prev  = cookie;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        local->op_ret   = -1;
+                        if (op_errno != ENOENT && op_errno != EACCES) {
+                                local->need_selfheal = 1;
+                        }
+
+
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "rmdir on %s for %s failed (%s)",
+                                prev->this->name, local->loc.path,
+                                strerror (op_errno));
+                        goto unlock;
+                }
+
+                dht_iatt_merge (this, &local->preparent, preparent, prev->this);
+                dht_iatt_merge (this, &local->postparent, postparent,
+                                prev->this);
+
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+               if (local->need_selfheal) {
+                        local->layout =
+                                dht_layout_get (this, local->loc.inode);
+
+                        /* TODO: neater interface needed below */
+                        local->stbuf.ia_type = local->loc.inode->ia_type;
+
+                        uuid_copy (local->gfid, local->loc.inode->gfid);
+                        dht_selfheal_restore (frame, dht_rmdir_selfheal_cbk,
+                                              &local->loc, local->layout);
+               } else {
+
+                        if (local->loc.parent) {
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->preparent,
+                                                           0);
+
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->postparent,
+                                                           1);
+                        }
+
+                        DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
+                                          local->op_errno, &local->preparent,
+                                          &local->postparent, NULL);
+               }
+        }
 
         return 0;
 }
@@ -3856,11 +4340,12 @@ dht_rmdir_selfheal_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 dht_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int op_ret, int op_errno, struct iatt *preparent,
-               struct iatt *postparent)
+               struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
         call_frame_t *prev = NULL;
+        int           done = 0;
 
         local = frame->local;
         prev  = cookie;
@@ -3893,7 +4378,16 @@ unlock:
 
 
         this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
+
+        /* if local->hashed_subvol, we are yet to wind to hashed_subvol. */
+        if (local->hashed_subvol && (this_call_cnt == 1)) {
+                done = 1;
+        } else if (!local->hashed_subvol && !this_call_cnt) {
+                done = 1;
+        }
+
+
+        if (done) {
                 if (local->need_selfheal && local->fop_succeeded) {
                         local->layout =
                                 dht_layout_get (this, local->loc.inode);
@@ -3904,15 +4398,34 @@ unlock:
                         uuid_copy (local->gfid, local->loc.inode->gfid);
                         dht_selfheal_restore (frame, dht_rmdir_selfheal_cbk,
                                               &local->loc, local->layout);
-                } else {
+                } else if (this_call_cnt) {
+                        /* If non-hashed subvol's have responded, proceed */
+
+                        local->need_selfheal = 0;
+                        STACK_WIND (frame, dht_rmdir_hashed_subvol_cbk,
+                                    local->hashed_subvol,
+                                    local->hashed_subvol->fops->rmdir,
+                                    &local->loc, local->flags, NULL);
+                } else if (!this_call_cnt) {
+                        /* All subvol's have responded, proceed */
+
                         if (local->loc.parent) {
-                                WIPE (&local->preparent);
-                                WIPE (&local->postparent);
+
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->preparent,
+                                                           0);
+
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->postparent,
+                                                           1);
+
                         }
 
                         DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
                                           local->op_errno, &local->preparent,
-                                          &local->postparent);
+                                          &local->postparent, NULL);
                 }
         }
 
@@ -3926,6 +4439,7 @@ dht_rmdir_do (call_frame_t *frame, xlator_t *this)
         dht_local_t  *local = NULL;
         dht_conf_t   *conf = NULL;
         int           i = 0;
+        xlator_t     *hashed_subvol = NULL;
 
         VALIDATE_OR_GOTO (this->private, err);
 
@@ -3937,18 +4451,41 @@ dht_rmdir_do (call_frame_t *frame, xlator_t *this)
 
         local->call_cnt = conf->subvolume_cnt;
 
+        /* first remove from non-hashed_subvol */
+        hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
+
+        if (!hashed_subvol) {
+                gf_log (this->name, GF_LOG_WARNING, "failed to get hashed "
+                        "subvol for %s",local->loc.path);
+        } else {
+                local->hashed_subvol = hashed_subvol;
+        }
+
+        /* When DHT has only 1 child */
+        if (conf->subvolume_cnt == 1) {
+                STACK_WIND (frame, dht_rmdir_hashed_subvol_cbk,
+                            conf->subvolumes[0],
+                            conf->subvolumes[0]->fops->rmdir,
+                            &local->loc, local->flags, NULL);
+                return 0;
+        }
+
         for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (hashed_subvol &&
+                    (hashed_subvol == conf->subvolumes[i]))
+                        continue;
+
                 STACK_WIND (frame, dht_rmdir_cbk,
                             conf->subvolumes[i],
                             conf->subvolumes[i]->fops->rmdir,
-                            &local->loc, local->flags);
+                            &local->loc, local->flags, NULL);
         }
 
         return 0;
 
 err:
         DHT_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent);
+                          &local->preparent, &local->postparent, NULL);
         return 0;
 }
 
@@ -3956,7 +4493,7 @@ err:
 int
 dht_rmdir_linkfile_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                int op_ret, int op_errno, struct iatt *preparent,
-                               struct iatt *postparent)
+                               struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t    *local = NULL;
         call_frame_t   *prev = NULL;
@@ -4004,6 +4541,7 @@ dht_rmdir_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         call_frame_t   *main_frame = NULL;
         dht_local_t    *main_local = NULL;
         int             this_call_cnt = 0;
+        dht_conf_t     *conf = this->private;
 
         local = frame->local;
         prev  = cookie;
@@ -4015,7 +4553,7 @@ dht_rmdir_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (op_ret != 0)
                 goto err;
 
-        if (check_is_linkfile (inode, stbuf, xattr) == 0) {
+        if (!check_is_linkfile (inode, stbuf, xattr, conf->link_xattr_name)) {
                 main_local->op_ret  = -1;
                 main_local->op_errno = ENOTEMPTY;
 
@@ -4026,7 +4564,7 @@ dht_rmdir_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         STACK_WIND (frame, dht_rmdir_linkfile_unlink_cbk,
-                    src, src->fops->unlink, &local->loc);
+                    src, src->fops->unlink, &local->loc, 0, NULL);
         return 0;
 err:
 
@@ -4049,6 +4587,8 @@ dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
         call_frame_t       *lookup_frame = NULL;
         dht_local_t        *lookup_local = NULL;
         dht_local_t        *local = NULL;
+        dict_t             *xattrs = NULL;
+        dht_conf_t         *conf = this->private;
 
         local = frame->local;
 
@@ -4057,7 +4597,8 @@ dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
                         continue;
                 if (strcmp (trav->d_name, "..") == 0)
                         continue;
-                if (check_is_linkfile (NULL, (&trav->d_stat), trav->dict)) {
+                if (check_is_linkfile (NULL, (&trav->d_stat), trav->dict,
+                                              conf->link_xattr_name)) {
                         ret++;
                         continue;
                 }
@@ -4069,6 +4610,21 @@ dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
                 return 0;
         }
 
+        xattrs = dict_new ();
+        if (!xattrs) {
+                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
+                return -1;
+        }
+
+        ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to set linkto key"
+                        " in dict");
+                if (xattrs)
+                        dict_unref (xattrs);
+                return -1;
+        }
+
         list_for_each_entry (trav, &entries->list, list) {
                 if (strcmp (trav->d_name, ".") == 0)
                         continue;
@@ -4112,12 +4668,18 @@ dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
 
                 STACK_WIND (lookup_frame, dht_rmdir_lookup_cbk,
                             src, src->fops->lookup,
-                            &lookup_local->loc, NULL);
+                            &lookup_local->loc, xattrs);
                 ret++;
         }
 
+        if (xattrs)
+                dict_unref (xattrs);
+
         return ret;
 err:
+        if (xattrs)
+                dict_unref (xattrs);
+
         DHT_STACK_DESTROY (lookup_frame);
         return 0;
 }
@@ -4125,7 +4687,8 @@ err:
 
 int
 dht_rmdir_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, gf_dirent_t *entries)
+                        int op_ret, int op_errno, gf_dirent_t *entries,
+                        dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = -1;
@@ -4169,13 +4732,14 @@ dht_rmdir_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 dht_rmdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno, fd_t *fd)
+                       int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = -1;
         call_frame_t *prev = NULL;
         dict_t       *dict = NULL;
         int           ret = 0;
+        dht_conf_t   *conf = this->private;
 
         local = frame->local;
         prev  = cookie;
@@ -4185,8 +4749,10 @@ dht_rmdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         "opendir on %s for %s failed (%s)",
                         prev->this->name, local->loc.path,
                         strerror (op_errno));
-                local->op_ret = -1;
-                local->op_errno = op_errno;
+                if (op_errno != ENOENT) {
+                        local->op_ret = -1;
+                        local->op_errno = op_errno;
+                }
                 goto err;
         }
 
@@ -4197,12 +4763,11 @@ dht_rmdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto err;
         }
 
-        ret = dict_set_uint32 (dict,
-                               "trusted.glusterfs.dht.linkto", 256);
+        ret = dict_set_uint32 (dict, conf->link_xattr_name, 256);
         if (ret)
                 gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set 'trusted.glusterfs.dht.linkto' key",
-                        local->loc.path);
+                        "%s: failed to set '%s' key",
+                        local->loc.path, conf->link_xattr_name);
 
         STACK_WIND (frame, dht_rmdir_readdirp_cbk,
                     prev->this, prev->this->fops->readdirp,
@@ -4225,7 +4790,8 @@ err:
 
 
 int
-dht_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+dht_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+           dict_t *xdata)
 {
         dht_local_t  *local  = NULL;
         dht_conf_t   *conf = NULL;
@@ -4264,7 +4830,7 @@ dht_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
                 STACK_WIND (frame, dht_rmdir_opendir_cbk,
                             conf->subvolumes[i],
                             conf->subvolumes[i]->fops->opendir,
-                            loc, local->fd);
+                            loc, local->fd, NULL);
         }
 
         return 0;
@@ -4272,17 +4838,17 @@ dht_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
         DHT_STACK_UNWIND (rmdir, frame, -1, op_errno,
-                          NULL, NULL);
+                          NULL, NULL, NULL);
 
         return 0;
 }
 
 int
 dht_entrylk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno)
+                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        DHT_STACK_UNWIND (entrylk, frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (entrylk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -4290,7 +4856,7 @@ dht_entrylk_cbk (call_frame_t *frame, void *cookie,
 int
 dht_entrylk (call_frame_t *frame, xlator_t *this,
              const char *volume, loc_t *loc, const char *basename,
-             entrylk_cmd cmd, entrylk_type type)
+             entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -4300,7 +4866,6 @@ dht_entrylk (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
 
         local = dht_local_init (frame, loc, NULL, GF_FOP_ENTRYLK);
         if (!local) {
@@ -4320,13 +4885,13 @@ dht_entrylk (call_frame_t *frame, xlator_t *this,
 
         STACK_WIND (frame, dht_entrylk_cbk,
                     subvol, subvol->fops->entrylk,
-                    volume, loc, basename, cmd, type);
+                    volume, loc, basename, cmd, type, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (entrylk, frame, -1, op_errno);
+        DHT_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -4334,10 +4899,10 @@ err:
 
 int
 dht_fentrylk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno)
+                  xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        DHT_STACK_UNWIND (fentrylk, frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (fentrylk, frame, op_ret, op_errno, NULL);
         return 0;
 }
 
@@ -4345,7 +4910,7 @@ dht_fentrylk_cbk (call_frame_t *frame, void *cookie,
 int
 dht_fentrylk (call_frame_t *frame, xlator_t *this,
               const char *volume, fd_t *fd, const char *basename,
-              entrylk_cmd cmd, entrylk_type type)
+              entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -4364,13 +4929,13 @@ dht_fentrylk (call_frame_t *frame, xlator_t *this,
 
         STACK_WIND (frame, dht_fentrylk_cbk,
                     subvol, subvol->fops->fentrylk,
-                    volume, fd, basename, cmd, type);
+                    volume, fd, basename, cmd, type, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fentrylk, frame, -1, op_errno);
+        DHT_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -4379,16 +4944,21 @@ err:
 int
 dht_forget (xlator_t *this, inode_t *inode)
 {
-        uint64_t      tmp_layout = 0;
+        uint64_t        ctx_int = 0;
+        dht_inode_ctx_t *ctx    = NULL;
         dht_layout_t *layout = NULL;
 
-        inode_ctx_del (inode, this, &tmp_layout);
+        inode_ctx_del (inode, this, &ctx_int);
 
-        if (!tmp_layout)
+        if (!ctx_int)
                 return 0;
 
-        layout = (dht_layout_t *)(long)tmp_layout;
+        ctx = (dht_inode_ctx_t *) (long) ctx_int;
+
+        layout = ctx->layout;
+        ctx->layout = NULL;
         dht_layout_unref (this, layout);
+        GF_FREE (ctx);
 
         return 0;
 }
@@ -4474,7 +5044,11 @@ dht_notify (xlator_t *this, int event, void *data, ...)
                 if (conf->assert_no_child_down) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "Received CHILD_DOWN. Exiting");
-                        kill (getpid(), SIGTERM);
+                        if (conf->defrag) {
+                                gf_defrag_stop (conf->defrag, NULL);
+                        } else {
+                                kill (getpid(), SIGTERM);
+                        }
                 }
 
                 for (i = 0; i < conf->subvolume_cnt; i++) {
@@ -4572,15 +5146,7 @@ unlock:
            or wait for anything else. Just propagate blindly */
         if (have_heard_from_all) {
                 propagate = 1;
-                if (conf->defrag) {
-                        ret = pthread_create (&conf->defrag->th, NULL,
-                                              gf_defrag_start, this);
-                        if (ret) {
-                                conf->defrag = NULL;
-                                GF_FREE (conf->defrag);
-                                kill (getpid(), SIGTERM);
-                        }
-                }
+
         }
 
 
@@ -4602,6 +5168,19 @@ unlock:
                                 /* continue to check other events for CHILD_UP */
                         }
                 }
+
+                /* rebalance is started with assert_no_child_down. So we do
+                 * not need to handle CHILD_DOWN event here.
+                 */
+                if (conf->defrag) {
+                        ret = gf_thread_create (&conf->defrag->th, NULL,
+						gf_defrag_start, this);
+                        if (ret) {
+                                conf->defrag = NULL;
+                                GF_FREE (conf->defrag);
+                                kill (getpid(), SIGTERM);
+                        }
+                }
         }
 
         ret = 0;
@@ -4610,3 +5189,24 @@ unlock:
 
         return ret;
 }
+
+int
+dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this, dht_layout_t **layout)
+{
+        dht_inode_ctx_t         *ctx            = NULL;
+        int                      ret            = -1;
+
+        ret = dht_inode_ctx_get (inode, this, &ctx);
+
+        if (!ret && ctx) {
+                if (ctx->layout) {
+                        if (layout)
+                                *layout = ctx->layout;
+                        ret = 0;
+                } else {
+                        ret = -1;
+                }
+        }
+
+        return ret;
+}
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index d7689cc7f..5ccd66799 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -22,6 +13,8 @@
 #include "config.h"
 #endif
 
+#include <regex.h>
+
 #include "dht-mem-types.h"
 #include "libxlator.h"
 #include "syncop.h"
@@ -38,7 +31,8 @@
 
 typedef int (*dht_selfheal_dir_cbk_t) (call_frame_t *frame, void *cookie,
                                        xlator_t     *this,
-                                       int32_t       op_ret, int32_t op_errno);
+                                       int32_t       op_ret, int32_t op_errno,
+                                       dict_t *xdata);
 typedef int (*dht_defrag_cbk_fn_t) (xlator_t        *this, call_frame_t *frame,
                                     int              ret);
 
@@ -61,20 +55,38 @@ struct dht_layout {
                 uint32_t   start;
                 uint32_t   stop;
                 xlator_t  *xlator;
-        } list[0];
+        } list[];
 };
 typedef struct dht_layout  dht_layout_t;
 
+struct dht_stat_time {
+        uint32_t        atime;
+        uint32_t        atime_nsec;
+        uint32_t        ctime;
+        uint32_t        ctime_nsec;
+        uint32_t        mtime;
+        uint32_t        mtime_nsec;
+};
+
+typedef struct dht_stat_time dht_stat_time_t;
+
+struct dht_inode_ctx {
+        dht_layout_t    *layout;
+        dht_stat_time_t  time;
+};
+
+typedef struct dht_inode_ctx dht_inode_ctx_t;
+
 
 typedef enum {
         DHT_HASH_TYPE_DM,
+        DHT_HASH_TYPE_DM_USER,
 } dht_hashfn_type_t;
 
 /* rebalance related */
 struct dht_rebalance_ {
         xlator_t            *from_subvol;
         xlator_t            *target_node;
-        int32_t              wbflags;
         off_t                offset;
         size_t               size;
         int32_t              flags;
@@ -83,6 +95,7 @@ struct dht_rebalance_ {
         struct iovec        *vector;
         struct iatt          stbuf;
         dht_defrag_cbk_fn_t  target_op_fn;
+        dict_t              *xdata;
 };
 
 struct dht_local {
@@ -129,7 +142,6 @@ struct dht_local {
         struct {
                 uint32_t         hole_cnt;
                 uint32_t         overlaps_cnt;
-                uint32_t         missing;
                 uint32_t         down;
                 uint32_t         misc;
                 dht_selfheal_dir_cbk_t   dir_cbk;
@@ -142,6 +154,7 @@ struct dht_local {
         int32_t flags;
         mode_t  mode;
         dev_t   rdev;
+        mode_t  umask;
 
         /* need for file-info */
         char   *xattr_val;
@@ -149,6 +162,7 @@ struct dht_local {
 
         /* which xattr request? */
         char xsel[256];
+        int32_t alloc_len;
 
         char   *newpath;
 
@@ -165,7 +179,12 @@ struct dht_local {
 
         glusterfs_fop_t      fop;
 
+        gf_boolean_t     linked;
+        xlator_t        *link_subvol;
+
         struct dht_rebalance_ rebalance;
+        xlator_t        *first_up_subvol;
+
 };
 typedef struct dht_local dht_local_t;
 
@@ -193,14 +212,27 @@ enum gf_defrag_status_t {
         GF_DEFRAG_STATUS_STOPPED,
         GF_DEFRAG_STATUS_COMPLETE,
         GF_DEFRAG_STATUS_FAILED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED,
 };
 typedef enum gf_defrag_status_t gf_defrag_status_t;
 
+typedef struct gf_defrag_pattern_list gf_defrag_pattern_list_t;
+
+struct gf_defrag_pattern_list {
+        char                       path_pattern[256];
+        uint64_t                   size;
+        gf_defrag_pattern_list_t  *next;
+};
 
 struct gf_defrag_info_ {
         uint64_t                     total_files;
         uint64_t                     total_data;
         uint64_t                     num_files_lookedup;
+        uint64_t                     total_failures;
+        uint64_t                     skipped;
         gf_lock_t                    lock;
         int                          cmd;
         pthread_t                    th;
@@ -211,7 +243,9 @@ struct gf_defrag_info_ {
         pid_t                        pid;
         inode_t                     *root_inode;
         uuid_t                       node_uuid;
-
+        struct timeval               start_time;
+        gf_boolean_t                 stats;
+        gf_defrag_pattern_list_t    *defrag_pattern;
 };
 
 typedef struct gf_defrag_info_ gf_defrag_info_t;
@@ -227,8 +261,8 @@ struct dht_conf {
         gf_boolean_t   search_unhashed;
         int            gen;
         dht_du_t      *du_stats;
-        uint64_t       min_free_disk;
-	uint32_t       min_free_inodes;
+        double         min_free_disk;
+	double         min_free_inodes;
         char           disk_unit;
         int32_t        refresh_interval;
         gf_boolean_t   unhashed_sticky_bit;
@@ -245,14 +279,28 @@ struct dht_conf {
         /* Will be a global flag to control the layout spread count */
         uint32_t       dir_spread_cnt;
 
-	struct syncenv *env; /* The env pointer to the rebalance synctask */
-
         /* to keep track of nodes which are decomissioned */
         xlator_t     **decommissioned_bricks;
         int            decommission_in_progress;
+        int            decommission_subvols_cnt;
 
         /* defrag related */
         gf_defrag_info_t *defrag;
+
+        /* Request to filter directory entries in readdir request */
+
+        gf_boolean_t    readdir_optimize;
+
+        /* Support regex-based name reinterpretation. */
+        regex_t         rsync_regex;
+        gf_boolean_t    rsync_regex_valid;
+        regex_t         extra_regex;
+        gf_boolean_t    extra_regex_valid;
+
+        /* Support variable xattr names. */
+        char            *xattr_name;
+        char            *link_xattr_name;
+        char            *wild_xattr_name;
 };
 typedef struct dht_conf dht_conf_t;
 
@@ -276,20 +324,19 @@ typedef enum {
 
 #define ENTRY_MISSING(op_ret, op_errno) (op_ret == -1 && op_errno == ENOENT)
 
-#define is_revalidate(loc) (inode_ctx_get (loc->inode, this, NULL) == 0)
+#define is_revalidate(loc) (dht_inode_ctx_layout_get (loc->inode, this, NULL) == 0)
 
 #define is_last_call(cnt) (cnt == 0)
 
 #define DHT_MIGRATION_IN_PROGRESS 1
 #define DHT_MIGRATION_COMPLETED   2
 
-#define DHT_LINKFILE_KEY         "trusted.glusterfs.dht.linkto"
 #define DHT_LINKFILE_MODE        (S_ISVTX)
 
-#define check_is_linkfile(i,s,x) (                                      \
+#define check_is_linkfile(i,s,x,n) (                                      \
                 ((st_mode_from_ia ((s)->ia_prot, (s)->ia_type) & ~S_IFMT) \
-                 == DHT_LINKFILE_MODE) &&                               \
-                dict_get (x, DHT_LINKFILE_KEY))
+                 == DHT_LINKFILE_MODE) &&                                 \
+                dict_get (x, n))
 
 #define IS_DHT_MIGRATION_PHASE2(buf)  (                                 \
                 IA_ISREG ((buf)->ia_type) &&                            \
@@ -334,6 +381,25 @@ typedef enum {
                 dht_local_wipe (__xl, __local);         \
         } while (0)
 
+#define DHT_UPDATE_TIME(ctx_sec, ctx_nsec, new_sec, new_nsec, inode, post) do {\
+                int32_t sec = 0;                                        \
+                sec = new_sec;                                          \
+                LOCK (&inode->lock);                                    \
+                {                                                       \
+                        new_sec = max(new_sec, ctx_sec);                \
+                        if (sec < new_sec)                              \
+                                new_nsec = ctx_nsec;                    \
+                        if (sec == new_sec)                             \
+                                new_nsec = max (new_nsec, ctx_nsec);    \
+                        if (post) {                                     \
+                                ctx_sec = new_sec;                      \
+                                ctx_nsec = new_nsec;                    \
+                        }                                               \
+                }                                                       \
+                UNLOCK (&inode->lock);                                  \
+        } while (0)
+
+#define is_greater_time(a, an, b, bn) (((a) < (b)) || (((a) == (b)) && ((an) < (bn))))
 dht_layout_t                            *dht_layout_new (xlator_t *this, int cnt);
 dht_layout_t                            *dht_layout_get (xlator_t *this, inode_t *inode);
 dht_layout_t                            *dht_layout_for_subvol (xlator_t *this, xlator_t *subvol);
@@ -343,7 +409,7 @@ int                                      dht_layout_normalize (xlator_t *this, l
 int dht_layout_anomalies (xlator_t      *this, loc_t *loc, dht_layout_t *layout,
                           uint32_t      *holes_p, uint32_t *overlaps_p,
                           uint32_t      *missing_p, uint32_t *down_p,
-                          uint32_t      *misc_p);
+                          uint32_t      *misc_p, uint32_t *no_space_p);
 int dht_layout_dir_mismatch (xlator_t   *this, dht_layout_t *layout,
                              xlator_t   *subvol, loc_t *loc, dict_t *xattr);
 
@@ -377,12 +443,14 @@ int dht_iatt_merge (xlator_t                 *this, struct iatt *to, struct iatt
 xlator_t *dht_subvol_get_hashed (xlator_t *this, loc_t *loc);
 xlator_t *dht_subvol_get_cached (xlator_t *this, inode_t *inode);
 xlator_t *dht_subvol_next (xlator_t *this, xlator_t *prev);
+xlator_t *dht_subvol_next_available (xlator_t *this, xlator_t *prev);
 int       dht_subvol_cnt (xlator_t *this, xlator_t *subvol);
 
-int dht_hash_compute (int type, const char *name, uint32_t *hash_p);
+int dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p);
 
 int dht_linkfile_create (call_frame_t    *frame, fop_mknod_cbk_t linkfile_cbk,
-                         xlator_t        *tovol, xlator_t *fromvol, loc_t *loc);
+                         xlator_t        *this, xlator_t *tovol,
+                         xlator_t        *fromvol, loc_t *loc);
 int                                       dht_lookup_directory (call_frame_t *frame, xlator_t *this, loc_t *loc);
 int                                       dht_lookup_everywhere (call_frame_t *frame, xlator_t *this, loc_t *loc);
 int
@@ -397,17 +465,15 @@ dht_selfheal_restore (call_frame_t       *frame, dht_selfheal_dir_cbk_t cbk,
 int
 dht_layout_sort_volname (dht_layout_t *layout);
 
-int dht_rename (call_frame_t *frame, xlator_t *this,
-                loc_t *oldloc, loc_t *newloc);
-
 int dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc);
 
 gf_boolean_t dht_is_subvol_filled (xlator_t *this, xlator_t *subvol);
-xlator_t *dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol);
+xlator_t *dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol,
+                                          dht_local_t *layout);
 int       dht_get_du_info_for_subvol (xlator_t *this, int subvol_idx);
 
 int dht_layout_preset (xlator_t *this, xlator_t *subvol, inode_t *inode);
-int           dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout);
+int           dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout);;
 void          dht_layout_unref (xlator_t *this, dht_layout_t *layout);
 dht_layout_t *dht_layout_ref (xlator_t *this, dht_layout_t *layout);
 xlator_t     *dht_first_up_subvol (xlator_t *this);
@@ -422,7 +488,8 @@ int                                     dht_rename_cleanup (call_frame_t *frame)
 int dht_rename_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                           int32_t           op_ret, int32_t op_errno,
                           inode_t          *inode, struct iatt *stbuf,
-                          struct iatt      *preparent, struct iatt *postparent);
+                          struct iatt      *preparent, struct iatt *postparent,
+                          dict_t *xdata);
 
 int dht_fix_directory_layout (call_frame_t *frame,
                               dht_selfheal_dir_cbk_t  dir_cbk,
@@ -445,73 +512,73 @@ int32_t dht_lookup (call_frame_t *frame,
 
 int32_t dht_stat (call_frame_t *frame,
                   xlator_t *this,
-                  loc_t    *loc);
+                  loc_t    *loc, dict_t *xdata);
 
 int32_t dht_fstat (call_frame_t *frame,
                    xlator_t *this,
-                   fd_t     *fd);
+                   fd_t     *fd, dict_t *xdata);
 
 int32_t dht_truncate (call_frame_t *frame,
                       xlator_t *this,
                       loc_t    *loc,
-                      off_t     offset);
+                      off_t     offset, dict_t *xdata);
 
 int32_t dht_ftruncate (call_frame_t *frame,
                        xlator_t *this,
                        fd_t     *fd,
-                       off_t     offset);
+                       off_t     offset, dict_t *xdata);
 
 int32_t dht_access (call_frame_t *frame,
                     xlator_t *this,
                     loc_t    *loc,
-                    int32_t   mask);
+                    int32_t   mask, dict_t *xdata);
 
 int32_t dht_readlink (call_frame_t *frame,
                       xlator_t *this,
                       loc_t    *loc,
-                      size_t    size);
+                      size_t    size, dict_t *xdata);
 
-int32_t dht_mknod (call_frame_t *frame, xlator_t *this,
-                   loc_t    *loc, mode_t mode, dev_t rdev, dict_t *params);
+int32_t dht_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata);
 
 int32_t dht_mkdir (call_frame_t *frame, xlator_t *this,
-                   loc_t    *loc, mode_t mode, dict_t *params);
+                   loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata);
 
 int32_t dht_unlink (call_frame_t *frame,
                     xlator_t *this,
-                    loc_t    *loc);
+                    loc_t    *loc, int xflag, dict_t *xdata);
 
 int32_t dht_rmdir (call_frame_t *frame, xlator_t *this,
-                   loc_t    *loc, int flags);
+                   loc_t    *loc, int flags, dict_t *xdata);
 
 int32_t dht_symlink (call_frame_t   *frame, xlator_t *this,
-                     const char *linkpath, loc_t *loc, dict_t *params);
+                     const char *linkpath, loc_t *loc, mode_t umask,
+                     dict_t *xdata);
 
 int32_t dht_rename (call_frame_t *frame,
                     xlator_t *this,
                     loc_t    *oldloc,
-                    loc_t    *newloc);
+                    loc_t    *newloc, dict_t *xdata);
 
 int32_t dht_link (call_frame_t *frame,
                   xlator_t *this,
                   loc_t    *oldloc,
-                  loc_t    *newloc);
+                  loc_t    *newloc, dict_t *xdata);
 
 int32_t dht_create (call_frame_t *frame, xlator_t *this,
                     loc_t    *loc, int32_t flags, mode_t mode,
-                    fd_t     *fd, dict_t *params);
+                    mode_t umask, fd_t     *fd, dict_t *params);
 
 int32_t dht_open (call_frame_t *frame,
                   xlator_t *this,
                   loc_t    *loc,
-                  int32_t   flags, fd_t *fd,
-                  int32_t   wbflags);
+                  int32_t   flags, fd_t *fd, dict_t *xdata);
 
 int32_t dht_readv (call_frame_t *frame,
                    xlator_t *this,
                    fd_t     *fd,
                    size_t    size,
-                   off_t     offset, uint32_t flags);
+                   off_t     offset, uint32_t flags, dict_t *xdata);
 
 int32_t dht_writev (call_frame_t      *frame,
                     xlator_t      *this,
@@ -520,87 +587,87 @@ int32_t dht_writev (call_frame_t      *frame,
                     int32_t        count,
                     off_t          offset,
                     uint32_t       flags,
-                    struct iobref *iobref);
+                    struct iobref *iobref, dict_t *xdata);
 
 int32_t dht_flush (call_frame_t *frame,
                    xlator_t *this,
-                   fd_t     *fd);
+                   fd_t     *fd, dict_t *xdata);
 
 int32_t dht_fsync (call_frame_t *frame,
                    xlator_t *this,
                    fd_t     *fd,
-                   int32_t   datasync);
+                   int32_t   datasync, dict_t *xdata);
 
 int32_t dht_opendir (call_frame_t *frame,
                      xlator_t *this,
-                     loc_t    *loc, fd_t *fd);
+                     loc_t    *loc, fd_t *fd, dict_t *xdata);
 
 int32_t dht_fsyncdir (call_frame_t *frame,
                       xlator_t *this,
                       fd_t     *fd,
-                      int32_t   datasync);
+                      int32_t   datasync, dict_t *xdata);
 
 int32_t dht_statfs (call_frame_t *frame,
                     xlator_t *this,
-                    loc_t    *loc);
+                    loc_t    *loc, dict_t *xdata);
 
 int32_t dht_setxattr (call_frame_t *frame,
                       xlator_t *this,
                       loc_t    *loc,
                       dict_t   *dict,
-                      int32_t   flags);
+                      int32_t   flags, dict_t *xdata);
 
 int32_t dht_getxattr (call_frame_t   *frame,
                       xlator_t   *this,
                       loc_t      *loc,
-                      const char *name);
+                      const char *name, dict_t *xdata);
 
 int32_t dht_fsetxattr (call_frame_t *frame,
                        xlator_t *this,
                        fd_t     *fd,
                        dict_t   *dict,
-                       int32_t   flags);
+                       int32_t   flags, dict_t *xdata);
 
 int32_t dht_fgetxattr (call_frame_t   *frame,
                        xlator_t   *this,
                        fd_t       *fd,
-                       const char *name);
+                       const char *name, dict_t *xdata);
 
 int32_t dht_removexattr (call_frame_t   *frame,
                          xlator_t   *this,
                          loc_t      *loc,
-                         const char *name);
+                         const char *name, dict_t *xdata);
 int32_t dht_fremovexattr (call_frame_t   *frame,
                           xlator_t   *this,
                           fd_t      *fd,
-                          const char *name);
+                          const char *name, dict_t *xdata);
 
 int32_t dht_lk (call_frame_t        *frame,
                 xlator_t        *this,
                 fd_t            *fd,
                 int32_t          cmd,
-                struct gf_flock *flock);
+                struct gf_flock *flock, dict_t *xdata);
 
 int32_t dht_inodelk (call_frame_t *frame, xlator_t *this,
                      const char      *volume, loc_t *loc, int32_t cmd,
-                     struct gf_flock *flock);
+                     struct gf_flock *flock, dict_t *xdata);
 
 int32_t dht_finodelk (call_frame_t        *frame, xlator_t *this,
                       const char      *volume, fd_t *fd, int32_t cmd,
-                      struct gf_flock *flock);
+                      struct gf_flock *flock, dict_t *xdata);
 
 int32_t dht_entrylk (call_frame_t    *frame, xlator_t *this,
                      const char  *volume, loc_t *loc, const char *basename,
-                     entrylk_cmd  cmd, entrylk_type type);
+                     entrylk_cmd  cmd, entrylk_type type, dict_t *xdata);
 
 int32_t dht_fentrylk (call_frame_t    *frame, xlator_t *this,
                       const char  *volume, fd_t *fd, const char *basename,
-                      entrylk_cmd  cmd, entrylk_type type);
+                      entrylk_cmd  cmd, entrylk_type type, dict_t *xdata);
 
 int32_t dht_readdir (call_frame_t  *frame,
                      xlator_t *this,
                      fd_t     *fd,
-                     size_t    size, off_t off);
+                     size_t    size, off_t off, dict_t *xdata);
 
 int32_t dht_readdirp (call_frame_t *frame,
                       xlator_t *this,
@@ -611,20 +678,29 @@ int32_t dht_xattrop (call_frame_t           *frame,
                      xlator_t           *this,
                      loc_t              *loc,
                      gf_xattrop_flags_t  flags,
-                     dict_t             *dict);
+                     dict_t             *dict, dict_t *xdata);
 
 int32_t dht_fxattrop (call_frame_t *frame,
                       xlator_t           *this,
                       fd_t               *fd,
                       gf_xattrop_flags_t  flags,
-                      dict_t             *dict);
+                      dict_t             *dict, dict_t *xdata);
 
 int32_t dht_forget (xlator_t *this, inode_t *inode);
 int32_t dht_setattr (call_frame_t  *frame, xlator_t *this, loc_t *loc,
-                     struct iatt   *stbuf, int32_t valid);
+                     struct iatt   *stbuf, int32_t valid, dict_t *xdata);
 int32_t dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      struct iatt  *stbuf, int32_t valid);
-
+                      struct iatt  *stbuf, int32_t valid, dict_t *xdata);
+int32_t dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd,
+		      int32_t mode, off_t offset, size_t len, dict_t *xdata);
+int32_t dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd,
+		    off_t offset, size_t len, dict_t *xdata);
+int32_t dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    off_t offset, size_t len, dict_t *xdata);
+
+int32_t dht_init (xlator_t *this);
+void    dht_fini (xlator_t *this);
+int     dht_reconfigure (xlator_t *this, dict_t *options);
 int32_t dht_notify (xlator_t *this, int32_t event, void *data, ...);
 
 /* definitions for nufa/switch */
@@ -647,11 +723,12 @@ int dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int op_ret, int op_errno,
                     fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                    struct iatt *preparent, struct iatt *postparent);
+                    struct iatt *preparent, struct iatt *postparent,
+                    dict_t *xdata);
 int dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int op_ret, int op_errno,
                      inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-                     struct iatt *postparent);
+                     struct iatt *postparent, dict_t *xdata);
 
 int
 gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict);
@@ -668,4 +745,43 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
 int
 dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                  int flag);
+int
+dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this,
+                          dht_layout_t **layout_int);
+int
+dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
+                          dht_layout_t* layout_int);
+int
+dht_inode_ctx_time_update (inode_t *inode, xlator_t *this, struct iatt *stat,
+                           int32_t update_ctx);
+
+int dht_inode_ctx_get (inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx);
+int dht_inode_ctx_set (inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx);
+int
+dht_dir_attr_heal (void *data);
+int
+dht_dir_attr_heal_done (int ret, call_frame_t *sync_frame, void *data);
+int
+dht_dir_has_layout (dict_t *xattr, char *name);
+gf_boolean_t
+dht_is_subvol_in_layout (dht_layout_t *layout, xlator_t *xlator);
+xlator_t *
+dht_subvol_with_free_space_inodes (xlator_t *this, xlator_t *subvol,
+                                   dht_layout_t *layout);
+xlator_t *
+dht_subvol_maxspace_nonzeroinode (xlator_t *this, xlator_t *subvol,
+                                  dht_layout_t *layout);
+int
+dht_linkfile_attr_heal (call_frame_t *frame, xlator_t *this);
+
+void
+dht_layout_dump (dht_layout_t  *layout, const char *prefix);
+int32_t
+dht_priv_dump (xlator_t *this);
+int32_t
+dht_inodectx_dump (xlator_t *this, inode_t *inode);
+
+int
+dht_inode_ctx_get1 (xlator_t *this, inode_t *inode, xlator_t **subvol);
+
 #endif/* _DHT_H */
diff --git a/xlators/cluster/dht/src/dht-diskusage.c b/xlators/cluster/dht/src/dht-diskusage.c
index 0200695d2..fe3955ecb 100644
--- a/xlators/cluster/dht/src/dht-diskusage.c
+++ b/xlators/cluster/dht/src/dht-diskusage.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -35,7 +26,8 @@
 
 int
 dht_du_info_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int op_ret, int op_errno, struct statvfs *statvfs)
+		 int op_ret, int op_errno, struct statvfs *statvfs,
+                 dict_t *xdata)
 {
 	dht_conf_t    *conf         = NULL;
 	call_frame_t  *prev          = NULL;
@@ -129,7 +121,7 @@ dht_get_du_info_for_subvol (xlator_t *this, int subvol_idx)
 	STACK_WIND (statfs_frame, dht_du_info_cbk,
 		    conf->subvolumes[subvol_idx],
 		    conf->subvolumes[subvol_idx]->fops->statfs,
-		    &tmp_loc);
+		    &tmp_loc, NULL);
 
 	return 0;
 err:
@@ -177,7 +169,7 @@ dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc)
 			STACK_WIND (statfs_frame, dht_du_info_cbk,
 				    conf->subvolumes[i],
 				    conf->subvolumes[i]->fops->statfs,
-				    &tmp_loc);
+				    &tmp_loc, NULL);
 		}
 
 		conf->last_stat_fetch.tv_sec = tv.tv_sec;
@@ -256,50 +248,167 @@ dht_is_subvol_filled (xlator_t *this, xlator_t *subvol)
 	return is_subvol_filled;
 }
 
+
+/*Get the best subvolume to create the file in*/
 xlator_t *
-dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol)
+dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol,
+                                dht_local_t *local)
 {
-	int         i = 0;
-	double      max = 0;
-	double      max_inodes = 0;
 	xlator_t   *avail_subvol = NULL;
 	dht_conf_t *conf = NULL;
+        dht_layout_t *layout = NULL;
+        loc_t      *loc = NULL;
 
 	conf = this->private;
-
-	LOCK (&conf->subvolume_lock);
+        if (!local)
+                goto out;
+        loc = &local->loc;
+        if (!local->layout) {
+                layout = dht_layout_get (this, loc->parent);
+
+                if (!layout) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "layout missing path=%s parent=%s",
+                                loc->path, uuid_utoa (loc->parent->gfid));
+                        goto out;
+                }
+        } else {
+                layout = dht_layout_ref (this, local->layout);
+        }
+
+        LOCK (&conf->subvolume_lock);
 	{
-		for (i = 0; i < conf->subvolume_cnt; i++) {
-			if (conf->disk_unit == 'p') {
-				if ((conf->du_stats[i].avail_percent > max)
-				    && (conf->du_stats[i].avail_inodes > max_inodes)) {
-					max = conf->du_stats[i].avail_percent;
-					max_inodes = conf->du_stats[i].avail_inodes;
-					avail_subvol = conf->subvolumes[i];
-				}
-			} else {
-				if ((conf->du_stats[i].avail_space > max)
-				    && (conf->du_stats[i].avail_inodes > max_inodes)) {
-					max = conf->du_stats[i].avail_space;
-					max_inodes = conf->du_stats[i].avail_inodes;
-					avail_subvol = conf->subvolumes[i];
-				}
+                avail_subvol = dht_subvol_with_free_space_inodes(this, subvol,
+                                                                 layout);
+                if(!avail_subvol)
+                {
+                        avail_subvol = dht_subvol_maxspace_nonzeroinode(this,
+                                                                        subvol,
+                                                                        layout);
+                }
 
-			}
-		}
 	}
 	UNLOCK (&conf->subvolume_lock);
-
+out:
 	if (!avail_subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no subvolume has enough free space and inodes to create");
+		gf_log (this->name,
+                        GF_LOG_DEBUG,
+			"no subvolume has enough free space and/or inodes\
+                         to create");
+                avail_subvol = subvol;
 	}
 
-	if ((max < conf->min_free_disk) && (max_inodes < conf->min_free_inodes))
-		avail_subvol = subvol;
+        if (layout)
+                dht_layout_unref (this, layout);
+	return avail_subvol;
+}
 
-	if (!avail_subvol)
-		avail_subvol = subvol;
+static inline
+int32_t dht_subvol_has_err (xlator_t *this, dht_layout_t *layout)
+{
+        int ret = -1;
+        int i   = 0;
+
+        if (!this || !layout)
+                goto out;
+
+        /* check if subvol has layout errors, before selecting it */
+        for (i = 0; i < layout->cnt; i++) {
+                if (!strcmp (layout->list[i].xlator->name, this->name) &&
+                     (layout->list[i].err != 0)) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+        ret = 0;
+out:
+        return ret;
+}
 
-	return avail_subvol;
+/*Get subvolume which has both space and inodes more than the min criteria*/
+xlator_t *
+dht_subvol_with_free_space_inodes(xlator_t *this, xlator_t *subvol,
+                                  dht_layout_t *layout)
+{
+        int i = 0;
+        double max = 0;
+        double max_inodes = 0;
+        int    ignore_subvol = 0;
+
+        xlator_t *avail_subvol = NULL;
+        dht_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        for(i=0; i < conf->subvolume_cnt; i++) {
+                /* check if subvol has layout errors, before selecting it */
+                ignore_subvol = dht_subvol_has_err (conf->subvolumes[i],
+                                                    layout);
+                if (ignore_subvol)
+                        continue;
+
+                if ((conf->disk_unit == 'p') &&
+                    (conf->du_stats[i].avail_percent > conf->min_free_disk) &&
+                    (conf->du_stats[i].avail_inodes  > conf->min_free_inodes)) {
+                        if ((conf->du_stats[i].avail_inodes > max_inodes) ||
+                            (conf->du_stats[i].avail_percent > max)) {
+                                max = conf->du_stats[i].avail_percent;
+                                max_inodes = conf->du_stats[i].avail_inodes;
+                                avail_subvol = conf->subvolumes[i];
+                        }
+                }
+
+                if ((conf->disk_unit != 'p') &&
+                    (conf->du_stats[i].avail_space > conf->min_free_disk) &&
+                    (conf->du_stats[i].avail_inodes  > conf->min_free_inodes)) {
+                        if ((conf->du_stats[i].avail_inodes > max_inodes) ||
+                            (conf->du_stats[i].avail_space > max)) {
+                                max = conf->du_stats[i].avail_space;
+                                max_inodes = conf->du_stats[i].avail_inodes;
+                                avail_subvol = conf->subvolumes[i];
+                        }
+                }
+        }
+
+        return avail_subvol;
+}
+
+
+/* Get subvol which has atleast one inode and maximum space */
+xlator_t *
+dht_subvol_maxspace_nonzeroinode (xlator_t *this, xlator_t *subvol,
+                                  dht_layout_t *layout)
+{
+        int         i = 0;
+        double      max = 0;
+        int         ignore_subvol = 0;
+
+        xlator_t   *avail_subvol = NULL;
+        dht_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                /* check if subvol has layout errors, before selecting it */
+                ignore_subvol = dht_subvol_has_err (conf->subvolumes[i],
+                                                    layout);
+                if (ignore_subvol)
+                        continue;
+
+                if (conf->disk_unit == 'p') {
+                        if ((conf->du_stats[i].avail_percent > max)
+                            && (conf->du_stats[i].avail_inodes > 0 )) {
+                                max = conf->du_stats[i].avail_percent;
+                                avail_subvol = conf->subvolumes[i];
+                        }
+               } else {
+                         if ((conf->du_stats[i].avail_space > max)
+                            && (conf->du_stats[i].avail_inodes > 0)) {
+                                 max = conf->du_stats[i].avail_space;
+                                 avail_subvol = conf->subvolumes[i];
+                         }
+               }
+        }
+
+        return avail_subvol;
 }
diff --git a/xlators/cluster/dht/src/dht-hashfn.c b/xlators/cluster/dht/src/dht-hashfn.c
index c8ae74172..656cf23a0 100644
--- a/xlators/cluster/dht/src/dht-hashfn.c
+++ b/xlators/cluster/dht/src/dht-hashfn.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -37,6 +28,7 @@ dht_hash_compute_internal (int type, const char *name, uint32_t *hash_p)
 
         switch (type) {
         case DHT_HASH_TYPE_DM:
+        case DHT_HASH_TYPE_DM_USER:
                 hash = gf_dm_hashfn (name, strlen (name));
                 break;
         default:
@@ -52,30 +44,68 @@ dht_hash_compute_internal (int type, const char *name, uint32_t *hash_p)
 }
 
 
-#define MAKE_RSYNC_FRIENDLY_NAME(rsync_frndly_name, name) do {          \
-                rsync_frndly_name = (char *) name;                      \
-                if (name[0] == '.') {                                   \
-                        char *dot   = 0;                                \
-                        int namelen = 0;                                \
-                                                                        \
-                        dot = strrchr (name, '.');                      \
-                        if (dot && dot > (name + 1) && *(dot + 1)) {    \
-                                namelen = (dot - name);                 \
-                                rsync_frndly_name = alloca (namelen);   \
-                                strncpy (rsync_frndly_name, name + 1,   \
-                                         namelen);                      \
-                                rsync_frndly_name[namelen - 1] = 0;     \
-                        }                                               \
-                }                                                       \
-        } while (0);
+static inline
+gf_boolean_t
+dht_munge_name (const char *original, char *modified, size_t len, regex_t *re)
+{
+        regmatch_t      matches[2];
+        size_t          new_len;
+
+        if (regexec(re,original,2,matches,0) != REG_NOMATCH) {
+                if (matches[1].rm_so != -1) {
+                        new_len = matches[1].rm_eo - matches[1].rm_so;
+                        /* Equal would fail due to the NUL at the end. */
+                        if (new_len < len) {
+                                memcpy (modified,original+matches[1].rm_so,
+                                        new_len);
+                                modified[new_len] = '\0';
+                                return _gf_true;
+                        }
+                }
+        }
 
+        /* This is guaranteed safe because of how the dest was allocated. */
+        strcpy(modified,original);
+        return _gf_false;
+}
 
 int
-dht_hash_compute (int type, const char *name, uint32_t *hash_p)
+dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p)
 {
-        char     *rsync_friendly_name = NULL;
+        char            *rsync_friendly_name    = NULL;
+        dht_conf_t      *priv                   = this->private;
+        size_t           len                    = 0;
+        gf_boolean_t     munged                 = _gf_false;
+
+        /*
+         * It wouldn't be safe to use alloca in an inline function that doesn't
+         * actually get inlined, and it wouldn't be efficient to do a real
+         * allocation, so we use alloca here (if needed) and pass that to the
+         * inline.
+         */
+
+        if (priv->extra_regex_valid) {
+                len = strlen(name) + 1;
+                rsync_friendly_name = alloca(len);
+                munged = dht_munge_name (name, rsync_friendly_name, len,
+                                         &priv->extra_regex);
+        }
+
+        if (!munged && priv->rsync_regex_valid) {
+                len = strlen(name) + 1;
+                rsync_friendly_name = alloca(len);
+                gf_log (this->name, GF_LOG_TRACE, "trying regex for %s", name);
+                munged = dht_munge_name (name, rsync_friendly_name, len,
+                                         &priv->rsync_regex);
+                if (munged) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "munged down to %s", rsync_friendly_name);
+                }
+        }
 
-        MAKE_RSYNC_FRIENDLY_NAME (rsync_friendly_name, name);
+        if (!munged) {
+                rsync_friendly_name = (char *)name;
+        }
 
         return dht_hash_compute_internal (type, rsync_friendly_name, hash_p);
 }
diff --git a/xlators/cluster/dht/src/dht-helper.c b/xlators/cluster/dht/src/dht-helper.c
index ff540013e..311a48112 100644
--- a/xlators/cluster/dht/src/dht-helper.c
+++ b/xlators/cluster/dht/src/dht-helper.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -27,6 +18,28 @@
 #include "xlator.h"
 #include "dht-common.h"
 
+static inline int
+dht_inode_ctx_set1 (xlator_t *this, inode_t *inode, xlator_t *subvol)
+{
+        uint64_t tmp_subvol = 0;
+
+        tmp_subvol = (long)subvol;
+        return inode_ctx_set1 (inode, this, &tmp_subvol);
+}
+
+int
+dht_inode_ctx_get1 (xlator_t *this, inode_t *inode, xlator_t **subvol)
+{
+        int ret = -1;
+        uint64_t tmp_subvol = 0;
+
+        ret =  inode_ctx_get1 (inode, this, &tmp_subvol);
+        if (tmp_subvol && subvol)
+                *subvol = (xlator_t *)tmp_subvol;
+
+        return ret;
+}
+
 
 int
 dht_frame_return (call_frame_t *frame)
@@ -49,6 +62,43 @@ dht_frame_return (call_frame_t *frame)
 }
 
 
+static uint64_t
+dht_bits_for (uint64_t num)
+{
+	uint64_t bits = 0, ctrl = 1;
+
+	while (ctrl < num) {
+		ctrl *= 2;
+		bits ++;
+	}
+
+	return bits;
+}
+
+/*
+ * A slightly "updated" version of the algorithm described in the commit log
+ * is used here.
+ *
+ * The only enhancement is that:
+ *
+ * - The number of bits used by the backend filesystem for HUGE d_off which
+ *   is described as 63, and
+ * - The number of bits used by the d_off presented by the transformation
+ *   upwards which is described as 64, are both made "configurable."
+ */
+
+
+#define BACKEND_D_OFF_BITS 63
+#define PRESENT_D_OFF_BITS 63
+
+#define ONE 1ULL
+#define MASK (~0ULL)
+#define PRESENT_MASK (MASK >> (64 - PRESENT_D_OFF_BITS))
+#define BACKEND_MASK (MASK >> (64 - BACKEND_D_OFF_BITS))
+
+#define TOP_BIT (ONE << (PRESENT_D_OFF_BITS - 1))
+#define SHIFT_BITS (max (0, (BACKEND_D_OFF_BITS - PRESENT_D_OFF_BITS + 1)))
+
 int
 dht_itransform (xlator_t *this, xlator_t *subvol, uint64_t x, uint64_t *y_p)
 {
@@ -56,6 +106,9 @@ dht_itransform (xlator_t *this, xlator_t *subvol, uint64_t x, uint64_t *y_p)
         int         cnt = 0;
         int         max = 0;
         uint64_t    y = 0;
+        uint64_t    hi_mask = 0;
+        uint64_t    off_mask = 0;
+        int         max_bits = 0;
 
         if (x == ((uint64_t) -1)) {
                 y = (uint64_t) -1;
@@ -69,7 +122,23 @@ dht_itransform (xlator_t *this, xlator_t *subvol, uint64_t x, uint64_t *y_p)
         max = conf->subvolume_cnt;
         cnt = dht_subvol_cnt (this, subvol);
 
-        y = ((x * max) + cnt);
+	if (max == 1) {
+		y = x;
+		goto out;
+	}
+
+        max_bits = dht_bits_for (max);
+
+        hi_mask = ~(PRESENT_MASK >> (max_bits + 1));
+
+        if (x & hi_mask) {
+                /* HUGE d_off */
+                off_mask = MASK << max_bits;
+                y = TOP_BIT | ((x >> SHIFT_BITS) & off_mask) | cnt;
+        } else {
+                /* small d_off */
+                y = ((x * max) + cnt);
+        }
 
 out:
         if (y_p)
@@ -89,7 +158,7 @@ dht_filter_loc_subvol_key (xlator_t *this, loc_t *loc, loc_t *new_loc,
         int            ret       = 0; /* not found */
 
         /* Why do other tasks if first required 'char' itself is not there */
-        if (!loc->name || !strchr (loc->name, '@'))
+        if (!new_loc || !loc || !loc->name || !strchr (loc->name, '@'))
                 goto out;
 
         trav = this->children;
@@ -129,10 +198,8 @@ dht_filter_loc_subvol_key (xlator_t *this, loc_t *loc, loc_t *new_loc,
 out:
         if (!ret) {
                 /* !success */
-                if (new_path)
-                        GF_FREE (new_path);
-                if (new_name)
-                        GF_FREE (new_name);
+                GF_FREE (new_path);
+                GF_FREE (new_name);
         }
         return ret;
 }
@@ -146,16 +213,38 @@ dht_deitransform (xlator_t *this, uint64_t y, xlator_t **subvol_p,
         int         max = 0;
         uint64_t    x = 0;
         xlator_t   *subvol = 0;
+        int         max_bits = 0;
+        uint64_t    off_mask = 0;
+        uint64_t    host_mask = 0;
 
         if (!this->private)
-                goto out;
+                return -1;
 
         conf = this->private;
         max = conf->subvolume_cnt;
 
-        cnt = y % max;
-        x   = y / max;
+	if (max == 1) {
+		x = y;
+		cnt = 0;
+		goto out;
+	}
+
+        if (y & TOP_BIT) {
+                /* HUGE d_off */
+                max_bits = dht_bits_for (max);
+                off_mask = (MASK << max_bits);
+                host_mask = ~(off_mask);
+
+                x = ((y & ~TOP_BIT) & off_mask) << SHIFT_BITS;
+
+                cnt = y & host_mask;
+	} else {
+                /* small d_off */
+                cnt = y % max;
+                x = y / max;
+        }
 
+out:
         subvol = conf->subvolumes[cnt];
 
         if (subvol_p)
@@ -164,7 +253,6 @@ dht_deitransform (xlator_t *this, uint64_t y, xlator_t **subvol_p,
         if (x_p)
                 *x_p = x;
 
-out:
         return 0;
 }
 
@@ -215,16 +303,11 @@ dht_local_wipe (xlator_t *this, dht_local_t *local)
                 local->selfheal.layout = NULL;
         }
 
-        if (local->newpath) {
-                GF_FREE (local->newpath);
-        }
+        GF_FREE (local->newpath);
 
-        if (local->key) {
-                GF_FREE (local->key);
-        }
+        GF_FREE (local->key);
 
-        if (local->rebalance.vector)
-                GF_FREE (local->rebalance.vector);
+        GF_FREE (local->rebalance.vector);
 
         if (local->rebalance.iobref)
                 iobref_unref (local->rebalance.iobref);
@@ -279,20 +362,6 @@ out:
         return local;
 }
 
-
-char *
-basestr (const char *str)
-{
-        char *basestr = NULL;
-
-        basestr = strrchr (str, '/');
-        if (basestr)
-                basestr ++;
-
-        return basestr;
-}
-
-
 xlator_t *
 dht_first_up_subvol (xlator_t *this)
 {
@@ -444,7 +513,36 @@ out:
         return next;
 }
 
+/* This func wraps around, if prev is actually the last subvol.
+ */
+xlator_t *
+dht_subvol_next_available (xlator_t *this, xlator_t *prev)
+{
+        dht_conf_t *conf = NULL;
+        int         i = 0;
+        xlator_t   *next = NULL;
+
+        conf = this->private;
+        if (!conf)
+                goto out;
 
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (conf->subvolumes[i] == prev) {
+                        /* if prev is last in conf->subvolumes, then wrap
+                         * around.
+                         */
+                        if ((i + 1) < conf->subvolume_cnt) {
+                                next = conf->subvolumes[i + 1];
+                        } else {
+                                next = conf->subvolumes[0];
+                        }
+                        break;
+                }
+        }
+
+out:
+        return next;
+}
 int
 dht_subvol_cnt (xlator_t *this, xlator_t *subvol)
 {
@@ -636,20 +734,36 @@ dht_migration_complete_check_task (void *data)
         call_frame_t *frame    = NULL;
         loc_t         tmp_loc  = {0,};
         char         *path     = NULL;
+        dht_conf_t   *conf     = NULL;
+        inode_t      *inode    = NULL;
+        fd_t         *iter_fd  = NULL;
+        uint64_t      tmp_subvol = 0;
+        int           open_failed = 0;
 
         this  = THIS;
         frame = data;
         local = frame->local;
+        conf = this->private;
 
         src_node = local->cached_subvol;
 
-        /* getxattr on cached_subvol for 'linkto' value */
-        if (!local->loc.inode)
+        if (!local->loc.inode && !local->fd)
+                goto out;
+
+        inode = (!local->fd) ? local->loc.inode : local->fd->inode;
+
+        /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
+         * as root:root. If a fd is already open, access check wont be done*/
+
+        if (!local->loc.inode) {
                 ret = syncop_fgetxattr (src_node, local->fd, &dict,
-                                        DHT_LINKFILE_KEY);
-        else
+                                        conf->link_xattr_name);
+        } else {
+                SYNCTASK_SETID (0, 0);
                 ret = syncop_getxattr (src_node, &local->loc, &dict,
-                                       DHT_LINKFILE_KEY);
+                                       conf->link_xattr_name);
+                SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+        }
 
         if (!ret)
                 dst_node = dht_linkfile_subvol (this, NULL, NULL, dict);
@@ -700,10 +814,7 @@ dht_migration_complete_check_task (void *data)
         /* update inode ctx (the layout) */
         dht_layout_unref (this, local->layout);
 
-        if (!local->loc.inode)
-                ret = dht_layout_preset (this, dst_node, local->fd->inode);
-        else
-                ret = dht_layout_preset (this, dst_node, local->loc.inode);
+        ret = dht_layout_preset (this, dst_node, inode);
         if (ret != 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "%s: could not set preset layout for subvol %s",
@@ -721,10 +832,7 @@ dht_migration_complete_check_task (void *data)
                 goto out;
         }
 
-        if (!local->loc.inode)
-                ret = dht_layout_set (this, local->fd->inode, layout);
-        else
-                ret = dht_layout_set (this, local->loc.inode, layout);
+        ret = dht_layout_set (this, inode, layout);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "%s: failed to set the new layout",
@@ -735,43 +843,46 @@ dht_migration_complete_check_task (void *data)
         local->cached_subvol = dst_node;
         ret = 0;
 
-        if (!local->fd)
+        /* once we detect the migration complete, the inode-ctx2 is no more
+           required.. delete the ctx and also, it means, open() already
+           done on all the fd of inode */
+        ret = inode_ctx_reset1 (inode, this, &tmp_subvol);
+        if (tmp_subvol)
                 goto out;
 
-        /* once we detect the migration complete, the fd-ctx is no more
-           required.. delete the ctx, and do one extra 'fd_unref' for open fd */
-        ret = fd_ctx_del (local->fd, this, NULL);
-        if (!ret) {
-                fd_unref (local->fd);
-                ret = 0;
+        if (list_empty (&inode->fd_list))
                 goto out;
-        }
 
-        /* if 'local->fd' (ie, fd based operation), send a 'open()' on
-           destination if not already done */
-        if (local->loc.inode) {
-                ret = syncop_open (dst_node, &local->loc,
-                                   local->fd->flags, local->fd);
-        } else {
-                tmp_loc.inode = local->fd->inode;
-                inode_path (local->fd->inode, NULL, &path);
-                if (path)
-                        tmp_loc.path = path;
-                ret = syncop_open (dst_node, &tmp_loc,
-                                   local->fd->flags, local->fd);
-                if (path)
-                        GF_FREE (path);
+        /* perform open as root:root. There is window between linkfile
+         * creation(root:root) and setattr with the correct uid/gid
+         */
+        SYNCTASK_SETID(0, 0);
+
+        /* perform 'open()' on all the fd's present on the inode */
+        tmp_loc.inode = inode;
+        inode_path (inode, NULL, &path);
+        if (path)
+                tmp_loc.path = path;
+        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
+                if (fd_is_anonymous (iter_fd))
+                        continue;
 
+                ret = syncop_open (dst_node, &tmp_loc,
+                                   iter_fd->flags, iter_fd);
+                if (ret == -1) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to open "
+                                "the fd (%p, flags=0%o) on file %s @ %s",
+                                iter_fd, iter_fd->flags, path, dst_node->name);
+                        open_failed = 1;
+                }
         }
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to send open() on target file at %s",
-                        local->loc.path, dst_node->name);
+        GF_FREE (path);
+
+        SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+        if (open_failed) {
+                ret = -1;
                 goto out;
         }
-
-        /* need this unref for the fd on src_node */
-        fd_unref (local->fd);
         ret = 0;
 out:
 
@@ -782,11 +893,8 @@ int
 dht_rebalance_complete_check (xlator_t *this, call_frame_t *frame)
 {
         int         ret     = -1;
-        dht_conf_t *conf    = NULL;
-
-        conf = this->private;
 
-        ret = synctask_new (conf->env, dht_migration_complete_check_task,
+        ret = synctask_new (this->ctx->env, dht_migration_complete_check_task,
                             dht_migration_complete_check_done,
                             frame, frame);
         return ret;
@@ -818,20 +926,34 @@ dht_rebalance_inprogress_task (void *data)
         char         *path     = NULL;
         struct iatt   stbuf    = {0,};
         loc_t         tmp_loc  = {0,};
+        dht_conf_t   *conf     = NULL;
+        inode_t      *inode    = NULL;
+        fd_t         *iter_fd  = NULL;
+        int           open_failed = 0;
 
         this  = THIS;
         frame = data;
         local = frame->local;
+        conf = this->private;
 
         src_node = local->cached_subvol;
 
-        /* getxattr on cached_subvol for 'linkto' value */
-        if (local->loc.inode)
+        if (!local->loc.inode && !local->fd)
+                goto out;
+
+        inode = (!local->fd) ? local->loc.inode : local->fd->inode;
+
+        /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
+         * as root:root. If a fd is already open, access check wont be done*/
+        if (local->loc.inode) {
+                SYNCTASK_SETID (0, 0);
                 ret = syncop_getxattr (src_node, &local->loc, &dict,
-                                       DHT_LINKFILE_KEY);
-        else
+                                       conf->link_xattr_name);
+                SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+        } else {
                 ret = syncop_fgetxattr (src_node, local->fd, &dict,
-                                        DHT_LINKFILE_KEY);
+                                        conf->link_xattr_name);
+        }
 
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
@@ -873,34 +995,46 @@ dht_rebalance_inprogress_task (void *data)
 
         ret = 0;
 
-        if (!local->fd)
-                goto out;
+        if (list_empty (&inode->fd_list))
+                goto done;
+
+        /* perform open as root:root. There is window between linkfile
+         * creation(root:root) and setattr with the correct uid/gid
+         */
+        SYNCTASK_SETID (0, 0);
+
+        tmp_loc.inode = inode;
+        inode_path (inode, NULL, &path);
+        if (path)
+                tmp_loc.path = path;
+
+        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
+                if (fd_is_anonymous (iter_fd))
+                        continue;
 
-        if (local->loc.inode) {
-                ret = syncop_open (dst_node, &local->loc,
-                                   local->fd->flags, local->fd);
-        } else {
-                tmp_loc.inode = local->fd->inode;
-                inode_path (local->fd->inode, NULL, &path);
-                if (path)
-                        tmp_loc.path = path;
                 ret = syncop_open (dst_node, &tmp_loc,
-                                   local->fd->flags, local->fd);
-                if (path)
-                        GF_FREE (path);
+                                   iter_fd->flags, iter_fd);
+                if (ret == -1) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to send open "
+                                "the fd (%p, flags=0%o) on file %s @ %s",
+                                iter_fd, iter_fd->flags, path, dst_node->name);
+                        open_failed = 1;
+                }
         }
+        GF_FREE (path);
 
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to send open() on target file at %s",
-                        local->loc.path, dst_node->name);
+        SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+
+        if (open_failed) {
+                ret = -1;
                 goto out;
         }
 
-        ret = fd_ctx_set (local->fd, this, (uint64_t)(long)dst_node);
+done:
+        ret = dht_inode_ctx_set1 (this, inode, dst_node);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to set fd-ctx target file at %s",
+                        "%s: failed to set inode-ctx target file at %s",
                         local->loc.path, dst_node->name);
                 goto out;
         }
@@ -915,12 +1049,99 @@ dht_rebalance_in_progress_check (xlator_t *this, call_frame_t *frame)
 {
 
         int         ret     = -1;
-        dht_conf_t *conf    = NULL;
 
-        conf = this->private;
-
-        ret = synctask_new (conf->env, dht_rebalance_inprogress_task,
+        ret = synctask_new (this->ctx->env, dht_rebalance_inprogress_task,
                             dht_inprogress_check_done,
                             frame, frame);
         return ret;
 }
+
+int
+dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
+                          dht_layout_t *layout_int)
+{
+        dht_inode_ctx_t         *ctx            = NULL;
+        int                      ret            = -1;
+
+        ret = dht_inode_ctx_get (inode, this, &ctx);
+        if (!ret && ctx) {
+                ctx->layout = layout_int;
+        } else {
+                ctx = GF_CALLOC (1, sizeof (*ctx), gf_dht_mt_inode_ctx_t);
+                if (!ctx)
+                        return ret;
+                ctx->layout = layout_int;
+        }
+
+        ret = dht_inode_ctx_set (inode, this, ctx);
+
+        return ret;
+}
+
+int
+dht_inode_ctx_time_update (inode_t *inode, xlator_t *this, struct iatt *stat,
+                           int32_t post)
+{
+        dht_inode_ctx_t         *ctx            = NULL;
+        dht_stat_time_t         *time           = 0;
+        int                      ret            = -1;
+
+        GF_VALIDATE_OR_GOTO (this->name, stat, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+
+        ret = dht_inode_ctx_get (inode, this, &ctx);
+
+        if (ret) {
+                ctx = GF_CALLOC (1, sizeof (*ctx), gf_dht_mt_inode_ctx_t);
+                if (!ctx)
+                        return -1;
+        }
+
+        time = &ctx->time;
+
+        DHT_UPDATE_TIME(time->mtime, time->mtime_nsec,
+                        stat->ia_mtime, stat->ia_mtime_nsec, inode, post);
+        DHT_UPDATE_TIME(time->ctime, time->ctime_nsec,
+                        stat->ia_ctime, stat->ia_ctime_nsec, inode, post);
+        DHT_UPDATE_TIME(time->atime, time->atime_nsec,
+                        stat->ia_atime, stat->ia_atime_nsec, inode, post);
+
+        ret = dht_inode_ctx_set (inode, this, ctx);
+out:
+        return 0;
+}
+
+int
+dht_inode_ctx_get (inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx)
+{
+        int             ret     = -1;
+        uint64_t        ctx_int = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+
+        ret = inode_ctx_get (inode, this, &ctx_int);
+
+        if (ret)
+                return ret;
+
+        if (ctx)
+                *ctx = (dht_inode_ctx_t *) ctx_int;
+out:
+        return ret;
+}
+
+int dht_inode_ctx_set (inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx)
+{
+        int             ret = -1;
+        uint64_t        ctx_int = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+        GF_VALIDATE_OR_GOTO (this->name, ctx, out);
+
+        ctx_int = (long)ctx;
+        ret = inode_ctx_set (inode, this, &ctx_int);
+out:
+        return ret;
+}
diff --git a/xlators/cluster/dht/src/dht-inode-read.c b/xlators/cluster/dht/src/dht-inode-read.c
index 3ec6f4ba2..ece84151a 100644
--- a/xlators/cluster/dht/src/dht-inode-read.c
+++ b/xlators/cluster/dht/src/dht-inode-read.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -34,7 +25,7 @@ int dht_fsync2 (xlator_t *this, call_frame_t *frame, int ret);
 
 int
 dht_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno, fd_t *fd)
+              int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
@@ -61,7 +52,7 @@ dht_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 return 0;
 
 out:
-        DHT_STACK_UNWIND (open, frame, op_ret, op_errno, local->fd);
+        DHT_STACK_UNWIND (open, frame, op_ret, op_errno, local->fd, xdata);
 
         return 0;
 }
@@ -86,17 +77,17 @@ dht_open2 (xlator_t *this, call_frame_t *frame, int op_ret)
 
         STACK_WIND (frame, dht_open_cbk, subvol, subvol->fops->open,
                     &local->loc, local->rebalance.flags, local->fd,
-                    local->rebalance.wbflags);
+                    NULL);
         return 0;
 
 out:
-        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 int
 dht_open (call_frame_t *frame, xlator_t *this,
-          loc_t *loc, int flags, fd_t *fd, int wbflags)
+          loc_t *loc, int flags, fd_t *fd, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -120,30 +111,30 @@ dht_open (call_frame_t *frame, xlator_t *this,
                 goto err;
         }
 
-        local->rebalance.wbflags = wbflags;
         local->rebalance.flags = flags;
         local->call_cnt = 1;
 
         STACK_WIND (frame, dht_open_cbk, subvol, subvol->fops->open,
-                    loc, flags, fd, wbflags);
+                    loc, flags, fd, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (open, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
 
 int
 dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, struct iatt *stbuf)
+                   int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata)
 {
-        uint64_t      tmp_subvol = 0;
+        xlator_t     *subvol = 0;
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
         int           ret = -1;
+        inode_t      *inode = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -164,26 +155,28 @@ dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (local->call_cnt != 1)
                 goto out;
 
+        local->op_errno = op_errno;
         /* Check if the rebalance phase2 is true */
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
-                if (local->fd)
-                        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-                if (ret) {
+                inode = (local->fd) ? local->fd->inode : local->loc.inode;
+                ret = dht_inode_ctx_get1 (this, inode, &subvol);
+                if (!subvol) {
                         /* Phase 2 of migration */
                         local->rebalance.target_op_fn = dht_attr2;
                         ret = dht_rebalance_complete_check (this, frame);
+                        if (!ret)
+                                return 0;
                 } else {
                         /* value is already set in fd_ctx, that means no need
                            to check for whether its complete or not. */
                         dht_attr2 (this, frame, 0);
-                }
-                if (!ret)
                         return 0;
+                }
         }
 
 out:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (stat, frame, op_ret, op_errno, stbuf);
+        DHT_STACK_UNWIND (stat, frame, op_ret, op_errno, stbuf, xdata);
 err:
         return 0;
 }
@@ -208,21 +201,21 @@ dht_attr2 (xlator_t *this, call_frame_t *frame, int op_ret)
 
         if (local->fop == GF_FOP_FSTAT) {
                 STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->fstat, local->fd);
+                            subvol->fops->fstat, local->fd, NULL);
         } else {
                 STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->stat, &local->loc);
+                            subvol->fops->stat, &local->loc, NULL);
         }
         return 0;
 
 out:
-        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 int
 dht_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno, struct iatt *stbuf)
+              int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -257,14 +250,14 @@ out:
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt)) {
                 DHT_STACK_UNWIND (stat, frame, local->op_ret, local->op_errno,
-                                  &local->stbuf);
+                                  &local->stbuf, xdata);
         }
 err:
         return 0;
 }
 
 int
-dht_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+dht_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -300,7 +293,7 @@ dht_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 subvol = local->cached_subvol;
 
                 STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->stat, loc);
+                            subvol->fops->stat, loc, xdata);
 
                 return 0;
         }
@@ -312,21 +305,21 @@ dht_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
 
                 STACK_WIND (frame, dht_attr_cbk,
                             subvol, subvol->fops->stat,
-                            loc);
+                            loc, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
 
 
 int
-dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
+dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -360,7 +353,7 @@ dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
                 subvol = local->cached_subvol;
 
                 STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->fstat, fd);
+                            subvol->fops->fstat, fd, xdata);
 
                 return 0;
         }
@@ -371,14 +364,14 @@ dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
                 subvol = layout->list[i].xlator;
                 STACK_WIND (frame, dht_attr_cbk,
                             subvol, subvol->fops->fstat,
-                            fd);
+                            fd, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fstat, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -387,10 +380,12 @@ int
 dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int op_ret, int op_errno,
                struct iovec *vector, int count, struct iatt *stbuf,
-               struct iobref *iobref)
+               struct iobref *iobref, dict_t *xdata)
 {
         dht_local_t *local      = NULL;
         int          ret        = 0;
+        inode_t     *inode      = NULL;
+        xlator_t    *subvol = 0;
 
         local = frame->local;
         if (!local) {
@@ -406,25 +401,27 @@ dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno != ENOENT))
                 goto out;
 
+        local->op_errno = op_errno;
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
                 /* File would be migrated to other node */
-                ret = fd_ctx_get (local->fd, this, NULL);
-                if (ret) {
+                ret = dht_inode_ctx_get1 (this, inode, &subvol);
+                if (!subvol) {
                         local->rebalance.target_op_fn = dht_readv2;
                         ret = dht_rebalance_complete_check (this, frame);
+                        if (!ret)
+                                return 0;
                 } else {
                         /* value is already set in fd_ctx, that means no need
                            to check for whether its complete or not. */
                         dht_readv2 (this, frame, 0);
-                }
-                if (!ret)
                         return 0;
+                }
         }
 
 out:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count, stbuf,
-                          iobref);
+                          iobref, xdata);
 
         return 0;
 }
@@ -449,18 +446,18 @@ dht_readv2 (xlator_t *this, call_frame_t *frame, int op_ret)
 
         STACK_WIND (frame, dht_readv_cbk, subvol, subvol->fops->readv,
                     local->fd, local->rebalance.size, local->rebalance.offset,
-                    local->rebalance.flags);
+                    local->rebalance.flags, NULL);
 
         return 0;
 
 out:
-        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
+        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
         return 0;
 }
 
 int
 dht_readv (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, size_t size, off_t off, uint32_t flags)
+           fd_t *fd, size_t size, off_t off, uint32_t flags, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -491,31 +488,52 @@ dht_readv (call_frame_t *frame, xlator_t *this,
 
         STACK_WIND (frame, dht_readv_cbk,
                     subvol, subvol->fops->readv,
-                    fd, size, off, flags);
+                    fd, size, off, flags, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
+        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
 
         return 0;
 }
 
 int
 dht_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno)
+                int op_ret, int op_errno, dict_t *xdata)
 {
         int          ret = -1;
         dht_local_t *local = NULL;
+        xlator_t    *subvol = NULL;
+        call_frame_t *prev = NULL;
 
         local = frame->local;
+        prev = cookie;
 
+        if (!prev || !prev->this)
+                goto out;
         if (local->call_cnt != 1)
                 goto out;
+        if ((op_ret == -1) && (op_errno == ENOTCONN) &&
+            IA_ISDIR(local->loc.inode->ia_type)) {
+
+                subvol = dht_subvol_next_available (this, prev->this);
+                if (!subvol)
+                        goto out;
 
+                /* check if we are done with visiting every node */
+                if (subvol == local->cached_subvol) {
+                        goto out;
+                }
+
+                STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
+                            &local->loc, local->rebalance.flags, NULL);
+                return 0;
+        }
         if ((op_ret == -1) && (op_errno == ENOENT)) {
                 /* File would be migrated to other node */
+                local->op_errno = op_errno;
                 local->rebalance.target_op_fn = dht_access2;
                 ret = dht_rebalance_complete_check (frame->this, frame);
                 if (!ret)
@@ -523,7 +541,7 @@ dht_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
 out:
-        DHT_STACK_UNWIND (access, frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -546,18 +564,19 @@ dht_access2 (xlator_t *this, call_frame_t *frame, int op_ret)
         subvol = local->cached_subvol;
 
         STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
-                    &local->loc, local->rebalance.flags);
+                    &local->loc, local->rebalance.flags, NULL);
 
         return 0;
 
 out:
-        DHT_STACK_UNWIND (access, frame, -1, op_errno);
+        DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
         return 0;
 }
 
 
 int
-dht_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
+dht_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+            dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -586,13 +605,13 @@ dht_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
         }
 
         STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
-                    loc, mask);
+                    loc, mask, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (access, frame, -1, op_errno);
+        DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -600,10 +619,11 @@ err:
 
 int
 dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno)
+               int op_ret, int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           ret = -1;
+        dht_local_t  *local  = NULL;
+        inode_t      *inode  = NULL;
+        xlator_t     *subvol = 0;
 
         local = frame->local;
 
@@ -613,14 +633,14 @@ dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
 
         /* If context is set, then send flush() it to the destination */
-        ret = fd_ctx_get (local->fd, this, NULL);
-        if (!ret) {
+        dht_inode_ctx_get1 (this, inode, &subvol);
+        if (subvol) {
                 dht_flush2 (this, frame, 0);
                 return 0;
         }
 
 out:
-        DHT_STACK_UNWIND (flush, frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (flush, frame, op_ret, op_errno, xdata);
 
         return 0;
 }
@@ -630,14 +650,10 @@ dht_flush2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
 
         local = frame->local;
 
-        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
 
         if (!subvol)
                 subvol = local->cached_subvol;
@@ -645,20 +661,19 @@ dht_flush2 (xlator_t *this, call_frame_t *frame, int op_ret)
         local->call_cnt = 2; /* This is the second attempt */
 
         STACK_WIND (frame, dht_flush_cbk,
-                    subvol, subvol->fops->flush, local->fd);
+                    subvol, subvol->fops->flush, local->fd, NULL);
 
         return 0;
 }
 
 
 int
-dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
         dht_local_t  *local = NULL;
 
-
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
@@ -680,13 +695,13 @@ dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
         local->call_cnt = 1;
 
         STACK_WIND (frame, dht_flush_cbk,
-                    subvol, subvol->fops->flush, fd);
+                    subvol, subvol->fops->flush, fd, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (flush, frame, -1, op_errno);
+        DHT_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -694,17 +709,20 @@ err:
 
 int
 dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-               int op_errno, struct iatt *prebuf, struct iatt *postbuf)
+               int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+               dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
         int           ret = -1;
+        inode_t      *inode = NULL;
+        xlator_t     *subvol = 0;
 
         local = frame->local;
         prev = cookie;
 
         local->op_errno = op_errno;
-        if (op_ret == -1) {
+        if (op_ret == -1 && (op_errno != ENOENT)) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "subvolume %s returned -1 (%s)",
                         prev->this->name, strerror (op_errno));
@@ -719,8 +737,9 @@ dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
                 goto out;
         }
 
-        ret = fd_ctx_get (local->fd, this, NULL);
-        if (ret) {
+        local->op_errno = op_errno;
+        dht_inode_ctx_get1 (this, inode, &subvol);
+        if (!subvol) {
                 local->rebalance.target_op_fn = dht_fsync2;
 
                 /* Check if the rebalance phase1 is true */
@@ -735,17 +754,18 @@ dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
                 if (IS_DHT_MIGRATION_PHASE2 (postbuf)) {
                         ret = dht_rebalance_complete_check (this, frame);
                 }
+                if (!ret)
+                        return 0;
         } else {
                 dht_fsync2 (this, frame, 0);
-        }
-        if (!ret)
                 return 0;
+        }
 
 out:
         DHT_STRIP_PHASE1_FLAGS (postbuf);
         DHT_STRIP_PHASE1_FLAGS (prebuf);
         DHT_STACK_UNWIND (fsync, frame, op_ret, op_errno,
-                          prebuf, postbuf);
+                          prebuf, postbuf, xdata);
 
         return 0;
 }
@@ -755,34 +775,29 @@ dht_fsync2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
 
         local = frame->local;
 
-        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
-
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
         if (!subvol)
                 subvol = local->cached_subvol;
 
         local->call_cnt = 2; /* This is the second attempt */
 
         STACK_WIND (frame, dht_fsync_cbk, subvol, subvol->fops->fsync,
-                    local->fd, local->rebalance.flags);
+                    local->fd, local->rebalance.flags, NULL);
 
         return 0;
 }
 
 int
-dht_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync)
+dht_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
+           dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
         dht_local_t  *local = NULL;
 
-
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
@@ -800,13 +815,13 @@ dht_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync)
         subvol = local->cached_subvol;
 
         STACK_WIND (frame, dht_fsync_cbk, subvol, subvol->fops->fsync,
-                    fd, datasync);
+                    fd, datasync, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -817,9 +832,9 @@ err:
    phase 2 of migration */
 int
 dht_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int op_ret, int op_errno, struct gf_flock *flock)
+            int op_ret, int op_errno, struct gf_flock *flock, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (lk, frame, op_ret, op_errno, flock);
+        DHT_STACK_UNWIND (lk, frame, op_ret, op_errno, flock, xdata);
 
         return 0;
 }
@@ -827,7 +842,7 @@ dht_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 dht_lk (call_frame_t *frame, xlator_t *this,
-        fd_t *fd, int cmd, struct gf_flock *flock)
+        fd_t *fd, int cmd, struct gf_flock *flock, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -847,13 +862,13 @@ dht_lk (call_frame_t *frame, xlator_t *this,
 
         /* TODO: for rebalance, we need to preserve the fop arguments */
         STACK_WIND (frame, dht_lk_cbk, subvol, subvol->fops->lk, fd,
-                    cmd, flock);
+                    cmd, flock, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lk, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -861,7 +876,8 @@ err:
 /* Symlinks are currently not migrated, so no need for any check here */
 int
 dht_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, const char *path, struct iatt *stbuf)
+                  int op_ret, int op_errno, const char *path,
+                  struct iatt *stbuf, dict_t *xdata)
 {
         dht_local_t *local = NULL;
 
@@ -876,14 +892,15 @@ dht_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 err:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, stbuf);
+        DHT_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, stbuf, xdata);
 
         return 0;
 }
 
 
 int
-dht_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
+dht_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+              dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -911,13 +928,13 @@ dht_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
 
         STACK_WIND (frame, dht_readlink_cbk,
                     subvol, subvol->fops->readlink,
-                    loc, size);
+                    loc, size, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -928,16 +945,16 @@ err:
 
 int
 dht_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *dict)
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict);
+        DHT_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
 dht_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t flags, dict_t *dict)
+             gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -968,13 +985,13 @@ dht_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
         STACK_WIND (frame,
                     dht_xattrop_cbk,
                     subvol, subvol->fops->xattrop,
-                    loc, flags, dict);
+                    loc, flags, dict, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -982,16 +999,16 @@ err:
 
 int
 dht_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *dict)
+                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, dict);
+        DHT_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
 dht_fxattrop (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict)
+              fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -1011,13 +1028,13 @@ dht_fxattrop (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame,
                     dht_fxattrop_cbk,
                     subvol, subvol->fops->fxattrop,
-                    fd, flags, dict);
+                    fd, flags, dict, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -1025,17 +1042,17 @@ err:
 
 int
 dht_inodelk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno)
+                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        DHT_STACK_UNWIND (inodelk, frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (inodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int32_t
-dht_inodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *lock)
+dht_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+             loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -1067,31 +1084,31 @@ dht_inodelk (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame,
                     dht_inodelk_cbk,
                     subvol, subvol->fops->inodelk,
-                    volume, loc, cmd, lock);
+                    volume, loc, cmd, lock, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (inodelk, frame, -1, op_errno);
+        DHT_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
 
         return 0;
 }
 
 
 int
-dht_finodelk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno)
+dht_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        DHT_STACK_UNWIND (finodelk, frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (finodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
-dht_finodelk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock)
+dht_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+              fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -1109,16 +1126,14 @@ dht_finodelk (call_frame_t *frame, xlator_t *this,
         }
 
 
-        STACK_WIND (frame,
-                    dht_finodelk_cbk,
-                    subvol, subvol->fops->finodelk,
-                    volume, fd, cmd, lock);
+        STACK_WIND (frame, dht_finodelk_cbk, subvol, subvol->fops->finodelk,
+                    volume, fd, cmd, lock, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (finodelk, frame, -1, op_errno);
+        DHT_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
 
         return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-inode-write.c b/xlators/cluster/dht/src/dht-inode-write.c
index 2f59e7c94..4b3f3a049 100644
--- a/xlators/cluster/dht/src/dht-inode-write.c
+++ b/xlators/cluster/dht/src/dht-inode-write.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -28,16 +19,20 @@
 int dht_writev2 (xlator_t *this, call_frame_t *frame, int ret);
 int dht_truncate2 (xlator_t *this, call_frame_t *frame, int ret);
 int dht_setattr2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_fallocate2(xlator_t *this, call_frame_t *frame, int op_ret);
+int dht_discard2(xlator_t *this, call_frame_t *frame, int op_ret);
+int dht_zerofill2(xlator_t *this, call_frame_t *frame, int op_ret);
 
 int
 dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int op_ret, int op_errno, struct iatt *prebuf,
-                struct iatt *postbuf)
+                struct iatt *postbuf, dict_t *xdata)
 {
         dht_local_t *local = NULL;
         int          ret   = -1;
+        xlator_t    *subvol = NULL;
 
-        if (op_ret == -1) {
+        if (op_ret == -1 && (op_errno != ENOENT)) {
                 goto out;
         }
 
@@ -59,6 +54,7 @@ dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local->rebalance.target_op_fn = dht_writev2;
 
+        local->op_errno = op_errno;
         /* Phase 2 of migration */
         if (IS_DHT_MIGRATION_PHASE2 (postbuf)) {
                 ret = dht_rebalance_complete_check (this, frame);
@@ -71,8 +67,8 @@ dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
                 dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
 
-                ret = fd_ctx_get (local->fd, this, NULL);
-                if (!ret) {
+                ret = dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+                if (subvol) {
                         dht_writev2 (this, frame, 0);
                         return 0;
                 }
@@ -85,7 +81,8 @@ out:
         DHT_STRIP_PHASE1_FLAGS (postbuf);
         DHT_STRIP_PHASE1_FLAGS (prebuf);
 
-        DHT_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        DHT_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
 
         return 0;
 }
@@ -95,14 +92,10 @@ dht_writev2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
 
         local = frame->local;
 
-        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
 
         if (!subvol)
                 subvol = local->cached_subvol;
@@ -113,7 +106,7 @@ dht_writev2 (xlator_t *this, call_frame_t *frame, int op_ret)
                     subvol, subvol->fops->writev,
                     local->fd, local->rebalance.vector, local->rebalance.count,
                     local->rebalance.offset, local->rebalance.flags,
-                    local->rebalance.iobref);
+                    local->rebalance.iobref, NULL);
 
         return 0;
 }
@@ -121,7 +114,7 @@ dht_writev2 (xlator_t *this, call_frame_t *frame, int op_ret)
 int
 dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
             struct iovec *vector, int count, off_t off, uint32_t flags,
-            struct iobref *iobref)
+            struct iobref *iobref, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -156,13 +149,13 @@ dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         STACK_WIND (frame, dht_writev_cbk,
                     subvol, subvol->fops->writev,
-                    fd, vector, count, off, flags, iobref);
+                    fd, vector, count, off, flags, iobref, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -172,11 +165,13 @@ err:
 int
 dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int op_ret, int op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf)
+                  struct iatt *postbuf, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
         int           ret = -1;
+        xlator_t    *subvol = NULL;
+        inode_t      *inode = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -206,6 +201,7 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local->rebalance.target_op_fn = dht_truncate2;
 
+        local->op_errno = op_errno;
         /* Phase 2 of migration */
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
                 ret = dht_rebalance_complete_check (this, frame);
@@ -217,8 +213,9 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
                 dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
                 dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-                ret = fd_ctx_get (local->fd, this, NULL);
-                if (!ret) {
+                inode = (local->fd) ? local->fd->inode : local->loc.inode;
+                dht_inode_ctx_get1 (this, inode, &subvol);
+                if (subvol) {
                         dht_truncate2 (this, frame, 0);
                         return 0;
                 }
@@ -231,7 +228,7 @@ out:
         DHT_STRIP_PHASE1_FLAGS (postbuf);
         DHT_STRIP_PHASE1_FLAGS (prebuf);
         DHT_STACK_UNWIND (truncate, frame, op_ret, op_errno,
-                          prebuf, postbuf);
+                          prebuf, postbuf, xdata);
 err:
         return 0;
 }
@@ -242,16 +239,13 @@ dht_truncate2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
+        inode_t      *inode = NULL;
 
         local = frame->local;
 
-        if (local->fd)
-                ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+        inode = local->fd ? local->fd->inode : local->loc.inode;
 
+        dht_inode_ctx_get1 (this, inode, &subvol);
         if (!subvol)
                 subvol = local->cached_subvol;
 
@@ -260,18 +254,19 @@ dht_truncate2 (xlator_t *this, call_frame_t *frame, int op_ret)
         if (local->fop == GF_FOP_TRUNCATE) {
                 STACK_WIND (frame, dht_truncate_cbk, subvol,
                             subvol->fops->truncate, &local->loc,
-                            local->rebalance.offset);
+                            local->rebalance.offset, NULL);
         } else {
                 STACK_WIND (frame, dht_truncate_cbk, subvol,
                             subvol->fops->ftruncate, local->fd,
-                            local->rebalance.offset);
+                            local->rebalance.offset, NULL);
         }
 
         return 0;
 }
 
 int
-dht_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
+dht_truncate (call_frame_t *frame, xlator_t *th
author	Kotresh HR <khiremat@redhat.com>	2015-04-28 18:09:29 +0530
committer	Vijay Bellur <vbellur@redhat.com>	2015-05-04 02:29:19 -0700
commit	02223349a0a141d5bd09edcd571a703bb2e5a94d (patch)
tree	3dc5060f5ff8ecf1d6d668d29d2305e7bc2deeb1 /libglusterfs/src/timer.h
parent	20353cc323704292753ea6b7b0034362109fef76 (diff)